1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/ip_fib.h> 25 #include <net/ip6_fib.h> 26 #include <net/nexthop.h> 27 #include <net/fib_rules.h> 28 #include <net/ip_tunnels.h> 29 #include <net/l3mdev.h> 30 #include <net/addrconf.h> 31 #include <net/ndisc.h> 32 #include <net/ipv6.h> 33 #include <net/fib_notifier.h> 34 #include <net/switchdev.h> 35 36 #include "spectrum.h" 37 #include "core.h" 38 #include "reg.h" 39 #include "spectrum_cnt.h" 40 #include "spectrum_dpipe.h" 41 #include "spectrum_ipip.h" 42 #include "spectrum_mr.h" 43 #include "spectrum_mr_tcam.h" 44 #include "spectrum_router.h" 45 #include "spectrum_span.h" 46 47 struct mlxsw_sp_fib; 48 struct mlxsw_sp_vr; 49 struct mlxsw_sp_lpm_tree; 50 struct mlxsw_sp_rif_ops; 51 52 struct mlxsw_sp_rif { 53 struct list_head nexthop_list; 54 struct list_head neigh_list; 55 struct net_device *dev; /* NULL for underlay RIF */ 56 struct mlxsw_sp_fid *fid; 57 unsigned char addr[ETH_ALEN]; 58 int mtu; 59 u16 rif_index; 60 u16 vr_id; 61 const struct mlxsw_sp_rif_ops *ops; 62 struct mlxsw_sp *mlxsw_sp; 63 64 unsigned int counter_ingress; 65 bool counter_ingress_valid; 66 unsigned int counter_egress; 67 bool counter_egress_valid; 68 }; 69 70 struct mlxsw_sp_rif_params { 71 struct net_device *dev; 72 union { 73 u16 system_port; 74 u16 lag_id; 75 }; 76 u16 vid; 77 bool lag; 78 }; 79 80 struct mlxsw_sp_rif_subport { 81 struct mlxsw_sp_rif common; 82 refcount_t ref_count; 83 union { 84 u16 system_port; 85 u16 lag_id; 86 }; 87 u16 vid; 88 bool lag; 89 }; 90 91 struct mlxsw_sp_rif_ipip_lb { 92 struct mlxsw_sp_rif common; 93 struct mlxsw_sp_rif_ipip_lb_config lb_config; 94 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 95 u16 ul_rif_id; /* Reserved for Spectrum. */ 96 }; 97 98 struct mlxsw_sp_rif_params_ipip_lb { 99 struct mlxsw_sp_rif_params common; 100 struct mlxsw_sp_rif_ipip_lb_config lb_config; 101 }; 102 103 struct mlxsw_sp_rif_ops { 104 enum mlxsw_sp_rif_type type; 105 size_t rif_size; 106 107 void (*setup)(struct mlxsw_sp_rif *rif, 108 const struct mlxsw_sp_rif_params *params); 109 int (*configure)(struct mlxsw_sp_rif *rif); 110 void (*deconfigure)(struct mlxsw_sp_rif *rif); 111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 114 }; 115 116 struct mlxsw_sp_router_ops { 117 int (*init)(struct mlxsw_sp *mlxsw_sp); 118 }; 119 120 static struct mlxsw_sp_rif * 121 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 122 const struct net_device *dev); 123 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 124 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 125 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 126 struct mlxsw_sp_lpm_tree *lpm_tree); 127 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 128 const struct mlxsw_sp_fib *fib, 129 u8 tree_id); 130 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 131 const struct mlxsw_sp_fib *fib); 132 133 static unsigned int * 134 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 135 enum mlxsw_sp_rif_counter_dir dir) 136 { 137 switch (dir) { 138 case MLXSW_SP_RIF_COUNTER_EGRESS: 139 return &rif->counter_egress; 140 case MLXSW_SP_RIF_COUNTER_INGRESS: 141 return &rif->counter_ingress; 142 } 143 return NULL; 144 } 145 146 static bool 147 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 148 enum mlxsw_sp_rif_counter_dir dir) 149 { 150 switch (dir) { 151 case MLXSW_SP_RIF_COUNTER_EGRESS: 152 return rif->counter_egress_valid; 153 case MLXSW_SP_RIF_COUNTER_INGRESS: 154 return rif->counter_ingress_valid; 155 } 156 return false; 157 } 158 159 static void 160 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 161 enum mlxsw_sp_rif_counter_dir dir, 162 bool valid) 163 { 164 switch (dir) { 165 case MLXSW_SP_RIF_COUNTER_EGRESS: 166 rif->counter_egress_valid = valid; 167 break; 168 case MLXSW_SP_RIF_COUNTER_INGRESS: 169 rif->counter_ingress_valid = valid; 170 break; 171 } 172 } 173 174 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 175 unsigned int counter_index, bool enable, 176 enum mlxsw_sp_rif_counter_dir dir) 177 { 178 char ritr_pl[MLXSW_REG_RITR_LEN]; 179 bool is_egress = false; 180 int err; 181 182 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 183 is_egress = true; 184 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 185 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 186 if (err) 187 return err; 188 189 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 190 is_egress); 191 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 192 } 193 194 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 195 struct mlxsw_sp_rif *rif, 196 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 197 { 198 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 199 unsigned int *p_counter_index; 200 bool valid; 201 int err; 202 203 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 204 if (!valid) 205 return -EINVAL; 206 207 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 208 if (!p_counter_index) 209 return -EINVAL; 210 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 211 MLXSW_REG_RICNT_OPCODE_NOP); 212 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 213 if (err) 214 return err; 215 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 216 return 0; 217 } 218 219 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 220 unsigned int counter_index) 221 { 222 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 223 224 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 225 MLXSW_REG_RICNT_OPCODE_CLEAR); 226 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 227 } 228 229 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 230 struct mlxsw_sp_rif *rif, 231 enum mlxsw_sp_rif_counter_dir dir) 232 { 233 unsigned int *p_counter_index; 234 int err; 235 236 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 237 if (!p_counter_index) 238 return -EINVAL; 239 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 240 p_counter_index); 241 if (err) 242 return err; 243 244 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 245 if (err) 246 goto err_counter_clear; 247 248 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 249 *p_counter_index, true, dir); 250 if (err) 251 goto err_counter_edit; 252 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 253 return 0; 254 255 err_counter_edit: 256 err_counter_clear: 257 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 258 *p_counter_index); 259 return err; 260 } 261 262 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 263 struct mlxsw_sp_rif *rif, 264 enum mlxsw_sp_rif_counter_dir dir) 265 { 266 unsigned int *p_counter_index; 267 268 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 269 return; 270 271 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 272 if (WARN_ON(!p_counter_index)) 273 return; 274 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 275 *p_counter_index, false, dir); 276 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 277 *p_counter_index); 278 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 279 } 280 281 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 282 { 283 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 284 struct devlink *devlink; 285 286 devlink = priv_to_devlink(mlxsw_sp->core); 287 if (!devlink_dpipe_table_counter_enabled(devlink, 288 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 289 return; 290 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 291 } 292 293 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 294 { 295 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 296 297 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 298 } 299 300 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 301 302 struct mlxsw_sp_prefix_usage { 303 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 304 }; 305 306 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 307 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 308 309 static bool 310 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 311 struct mlxsw_sp_prefix_usage *prefix_usage2) 312 { 313 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 314 } 315 316 static void 317 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 318 struct mlxsw_sp_prefix_usage *prefix_usage2) 319 { 320 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 321 } 322 323 static void 324 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 325 unsigned char prefix_len) 326 { 327 set_bit(prefix_len, prefix_usage->b); 328 } 329 330 static void 331 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 332 unsigned char prefix_len) 333 { 334 clear_bit(prefix_len, prefix_usage->b); 335 } 336 337 struct mlxsw_sp_fib_key { 338 unsigned char addr[sizeof(struct in6_addr)]; 339 unsigned char prefix_len; 340 }; 341 342 enum mlxsw_sp_fib_entry_type { 343 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 344 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 345 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 346 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 347 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 348 349 /* This is a special case of local delivery, where a packet should be 350 * decapsulated on reception. Note that there is no corresponding ENCAP, 351 * because that's a type of next hop, not of FIB entry. (There can be 352 * several next hops in a REMOTE entry, and some of them may be 353 * encapsulating entries.) 354 */ 355 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 356 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 357 }; 358 359 struct mlxsw_sp_nexthop_group_info; 360 struct mlxsw_sp_nexthop_group; 361 struct mlxsw_sp_fib_entry; 362 363 struct mlxsw_sp_fib_node { 364 struct mlxsw_sp_fib_entry *fib_entry; 365 struct list_head list; 366 struct rhash_head ht_node; 367 struct mlxsw_sp_fib *fib; 368 struct mlxsw_sp_fib_key key; 369 }; 370 371 struct mlxsw_sp_fib_entry_decap { 372 struct mlxsw_sp_ipip_entry *ipip_entry; 373 u32 tunnel_index; 374 }; 375 376 static struct mlxsw_sp_fib_entry_priv * 377 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 378 { 379 struct mlxsw_sp_fib_entry_priv *priv; 380 381 if (!ll_ops->fib_entry_priv_size) 382 /* No need to have priv */ 383 return NULL; 384 385 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 386 if (!priv) 387 return ERR_PTR(-ENOMEM); 388 refcount_set(&priv->refcnt, 1); 389 return priv; 390 } 391 392 static void 393 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 394 { 395 kfree(priv); 396 } 397 398 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 399 { 400 refcount_inc(&priv->refcnt); 401 } 402 403 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 404 { 405 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 406 return; 407 mlxsw_sp_fib_entry_priv_destroy(priv); 408 } 409 410 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 411 struct mlxsw_sp_fib_entry_priv *priv) 412 { 413 if (!priv) 414 return; 415 mlxsw_sp_fib_entry_priv_hold(priv); 416 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 417 } 418 419 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 420 { 421 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 422 423 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 424 mlxsw_sp_fib_entry_priv_put(priv); 425 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 426 } 427 428 struct mlxsw_sp_fib_entry { 429 struct mlxsw_sp_fib_node *fib_node; 430 enum mlxsw_sp_fib_entry_type type; 431 struct list_head nexthop_group_node; 432 struct mlxsw_sp_nexthop_group *nh_group; 433 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 434 struct mlxsw_sp_fib_entry_priv *priv; 435 }; 436 437 struct mlxsw_sp_fib4_entry { 438 struct mlxsw_sp_fib_entry common; 439 struct fib_info *fi; 440 u32 tb_id; 441 u8 tos; 442 u8 type; 443 }; 444 445 struct mlxsw_sp_fib6_entry { 446 struct mlxsw_sp_fib_entry common; 447 struct list_head rt6_list; 448 unsigned int nrt6; 449 }; 450 451 struct mlxsw_sp_rt6 { 452 struct list_head list; 453 struct fib6_info *rt; 454 }; 455 456 struct mlxsw_sp_lpm_tree { 457 u8 id; /* tree ID */ 458 unsigned int ref_count; 459 enum mlxsw_sp_l3proto proto; 460 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 461 struct mlxsw_sp_prefix_usage prefix_usage; 462 }; 463 464 struct mlxsw_sp_fib { 465 struct rhashtable ht; 466 struct list_head node_list; 467 struct mlxsw_sp_vr *vr; 468 struct mlxsw_sp_lpm_tree *lpm_tree; 469 enum mlxsw_sp_l3proto proto; 470 const struct mlxsw_sp_router_ll_ops *ll_ops; 471 }; 472 473 struct mlxsw_sp_vr { 474 u16 id; /* virtual router ID */ 475 u32 tb_id; /* kernel fib table id */ 476 unsigned int rif_count; 477 struct mlxsw_sp_fib *fib4; 478 struct mlxsw_sp_fib *fib6; 479 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 480 struct mlxsw_sp_rif *ul_rif; 481 refcount_t ul_rif_refcnt; 482 }; 483 484 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 485 enum mlxsw_sp_l3proto proto) 486 { 487 return 0; 488 } 489 490 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 491 { 492 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 493 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 494 } 495 496 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 497 { 498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 499 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 500 } 501 502 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 503 { 504 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 505 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 506 } 507 508 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 509 510 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 511 struct mlxsw_sp_vr *vr, 512 enum mlxsw_sp_l3proto proto) 513 { 514 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 515 struct mlxsw_sp_lpm_tree *lpm_tree; 516 struct mlxsw_sp_fib *fib; 517 int err; 518 519 err = ll_ops->init(mlxsw_sp, vr->id, proto); 520 if (err) 521 return ERR_PTR(err); 522 523 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 524 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 525 if (!fib) 526 return ERR_PTR(-ENOMEM); 527 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 528 if (err) 529 goto err_rhashtable_init; 530 INIT_LIST_HEAD(&fib->node_list); 531 fib->proto = proto; 532 fib->vr = vr; 533 fib->lpm_tree = lpm_tree; 534 fib->ll_ops = ll_ops; 535 mlxsw_sp_lpm_tree_hold(lpm_tree); 536 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 537 if (err) 538 goto err_lpm_tree_bind; 539 return fib; 540 541 err_lpm_tree_bind: 542 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 543 err_rhashtable_init: 544 kfree(fib); 545 return ERR_PTR(err); 546 } 547 548 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 549 struct mlxsw_sp_fib *fib) 550 { 551 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 552 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 553 WARN_ON(!list_empty(&fib->node_list)); 554 rhashtable_destroy(&fib->ht); 555 kfree(fib); 556 } 557 558 static struct mlxsw_sp_lpm_tree * 559 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 560 { 561 static struct mlxsw_sp_lpm_tree *lpm_tree; 562 int i; 563 564 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 565 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 566 if (lpm_tree->ref_count == 0) 567 return lpm_tree; 568 } 569 return NULL; 570 } 571 572 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 573 const struct mlxsw_sp_router_ll_ops *ll_ops, 574 struct mlxsw_sp_lpm_tree *lpm_tree) 575 { 576 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 577 578 mlxsw_reg_xralta_pack(xralta_pl, true, 579 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 580 lpm_tree->id); 581 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 582 } 583 584 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 585 const struct mlxsw_sp_router_ll_ops *ll_ops, 586 struct mlxsw_sp_lpm_tree *lpm_tree) 587 { 588 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 589 590 mlxsw_reg_xralta_pack(xralta_pl, false, 591 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 592 lpm_tree->id); 593 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 594 } 595 596 static int 597 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 598 const struct mlxsw_sp_router_ll_ops *ll_ops, 599 struct mlxsw_sp_prefix_usage *prefix_usage, 600 struct mlxsw_sp_lpm_tree *lpm_tree) 601 { 602 char xralst_pl[MLXSW_REG_XRALST_LEN]; 603 u8 root_bin = 0; 604 u8 prefix; 605 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 606 607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 608 root_bin = prefix; 609 610 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 611 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 612 if (prefix == 0) 613 continue; 614 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 615 MLXSW_REG_RALST_BIN_NO_CHILD); 616 last_prefix = prefix; 617 } 618 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 619 } 620 621 static struct mlxsw_sp_lpm_tree * 622 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 623 const struct mlxsw_sp_router_ll_ops *ll_ops, 624 struct mlxsw_sp_prefix_usage *prefix_usage, 625 enum mlxsw_sp_l3proto proto) 626 { 627 struct mlxsw_sp_lpm_tree *lpm_tree; 628 int err; 629 630 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 631 if (!lpm_tree) 632 return ERR_PTR(-EBUSY); 633 lpm_tree->proto = proto; 634 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 635 if (err) 636 return ERR_PTR(err); 637 638 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 639 if (err) 640 goto err_left_struct_set; 641 memcpy(&lpm_tree->prefix_usage, prefix_usage, 642 sizeof(lpm_tree->prefix_usage)); 643 memset(&lpm_tree->prefix_ref_count, 0, 644 sizeof(lpm_tree->prefix_ref_count)); 645 lpm_tree->ref_count = 1; 646 return lpm_tree; 647 648 err_left_struct_set: 649 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 650 return ERR_PTR(err); 651 } 652 653 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 654 const struct mlxsw_sp_router_ll_ops *ll_ops, 655 struct mlxsw_sp_lpm_tree *lpm_tree) 656 { 657 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 658 } 659 660 static struct mlxsw_sp_lpm_tree * 661 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 662 struct mlxsw_sp_prefix_usage *prefix_usage, 663 enum mlxsw_sp_l3proto proto) 664 { 665 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 666 struct mlxsw_sp_lpm_tree *lpm_tree; 667 int i; 668 669 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 670 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 671 if (lpm_tree->ref_count != 0 && 672 lpm_tree->proto == proto && 673 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 674 prefix_usage)) { 675 mlxsw_sp_lpm_tree_hold(lpm_tree); 676 return lpm_tree; 677 } 678 } 679 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 680 } 681 682 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 683 { 684 lpm_tree->ref_count++; 685 } 686 687 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 688 struct mlxsw_sp_lpm_tree *lpm_tree) 689 { 690 const struct mlxsw_sp_router_ll_ops *ll_ops = 691 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 692 693 if (--lpm_tree->ref_count == 0) 694 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 695 } 696 697 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 698 699 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 700 { 701 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 702 struct mlxsw_sp_lpm_tree *lpm_tree; 703 u64 max_trees; 704 int err, i; 705 706 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 707 return -EIO; 708 709 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 710 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 711 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 712 sizeof(struct mlxsw_sp_lpm_tree), 713 GFP_KERNEL); 714 if (!mlxsw_sp->router->lpm.trees) 715 return -ENOMEM; 716 717 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 718 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 719 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 720 } 721 722 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 723 MLXSW_SP_L3_PROTO_IPV4); 724 if (IS_ERR(lpm_tree)) { 725 err = PTR_ERR(lpm_tree); 726 goto err_ipv4_tree_get; 727 } 728 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 729 730 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 731 MLXSW_SP_L3_PROTO_IPV6); 732 if (IS_ERR(lpm_tree)) { 733 err = PTR_ERR(lpm_tree); 734 goto err_ipv6_tree_get; 735 } 736 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 737 738 return 0; 739 740 err_ipv6_tree_get: 741 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 742 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 743 err_ipv4_tree_get: 744 kfree(mlxsw_sp->router->lpm.trees); 745 return err; 746 } 747 748 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 749 { 750 struct mlxsw_sp_lpm_tree *lpm_tree; 751 752 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 754 755 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 756 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 757 758 kfree(mlxsw_sp->router->lpm.trees); 759 } 760 761 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 762 { 763 return !!vr->fib4 || !!vr->fib6 || 764 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 765 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 766 } 767 768 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 769 { 770 struct mlxsw_sp_vr *vr; 771 int i; 772 773 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 774 vr = &mlxsw_sp->router->vrs[i]; 775 if (!mlxsw_sp_vr_is_used(vr)) 776 return vr; 777 } 778 return NULL; 779 } 780 781 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 782 const struct mlxsw_sp_fib *fib, u8 tree_id) 783 { 784 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 785 786 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 787 (enum mlxsw_reg_ralxx_protocol) fib->proto, 788 tree_id); 789 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 790 } 791 792 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 793 const struct mlxsw_sp_fib *fib) 794 { 795 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 796 797 /* Bind to tree 0 which is default */ 798 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 799 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 800 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 801 } 802 803 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 804 { 805 /* For our purpose, squash main, default and local tables into one */ 806 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 807 tb_id = RT_TABLE_MAIN; 808 return tb_id; 809 } 810 811 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 812 u32 tb_id) 813 { 814 struct mlxsw_sp_vr *vr; 815 int i; 816 817 tb_id = mlxsw_sp_fix_tb_id(tb_id); 818 819 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 820 vr = &mlxsw_sp->router->vrs[i]; 821 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 822 return vr; 823 } 824 return NULL; 825 } 826 827 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 828 u16 *vr_id) 829 { 830 struct mlxsw_sp_vr *vr; 831 int err = 0; 832 833 mutex_lock(&mlxsw_sp->router->lock); 834 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 835 if (!vr) { 836 err = -ESRCH; 837 goto out; 838 } 839 *vr_id = vr->id; 840 out: 841 mutex_unlock(&mlxsw_sp->router->lock); 842 return err; 843 } 844 845 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 846 enum mlxsw_sp_l3proto proto) 847 { 848 switch (proto) { 849 case MLXSW_SP_L3_PROTO_IPV4: 850 return vr->fib4; 851 case MLXSW_SP_L3_PROTO_IPV6: 852 return vr->fib6; 853 } 854 return NULL; 855 } 856 857 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 858 u32 tb_id, 859 struct netlink_ext_ack *extack) 860 { 861 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 862 struct mlxsw_sp_fib *fib4; 863 struct mlxsw_sp_fib *fib6; 864 struct mlxsw_sp_vr *vr; 865 int err; 866 867 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 868 if (!vr) { 869 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 870 return ERR_PTR(-EBUSY); 871 } 872 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 873 if (IS_ERR(fib4)) 874 return ERR_CAST(fib4); 875 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 876 if (IS_ERR(fib6)) { 877 err = PTR_ERR(fib6); 878 goto err_fib6_create; 879 } 880 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 881 MLXSW_SP_L3_PROTO_IPV4); 882 if (IS_ERR(mr4_table)) { 883 err = PTR_ERR(mr4_table); 884 goto err_mr4_table_create; 885 } 886 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 887 MLXSW_SP_L3_PROTO_IPV6); 888 if (IS_ERR(mr6_table)) { 889 err = PTR_ERR(mr6_table); 890 goto err_mr6_table_create; 891 } 892 893 vr->fib4 = fib4; 894 vr->fib6 = fib6; 895 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 896 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 897 vr->tb_id = tb_id; 898 return vr; 899 900 err_mr6_table_create: 901 mlxsw_sp_mr_table_destroy(mr4_table); 902 err_mr4_table_create: 903 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 904 err_fib6_create: 905 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 906 return ERR_PTR(err); 907 } 908 909 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 910 struct mlxsw_sp_vr *vr) 911 { 912 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 913 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 914 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 915 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 916 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 917 vr->fib6 = NULL; 918 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 919 vr->fib4 = NULL; 920 } 921 922 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 923 struct netlink_ext_ack *extack) 924 { 925 struct mlxsw_sp_vr *vr; 926 927 tb_id = mlxsw_sp_fix_tb_id(tb_id); 928 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 929 if (!vr) 930 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 931 return vr; 932 } 933 934 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 935 { 936 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 937 list_empty(&vr->fib6->node_list) && 938 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 939 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 940 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 941 } 942 943 static bool 944 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 945 enum mlxsw_sp_l3proto proto, u8 tree_id) 946 { 947 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 948 949 if (!mlxsw_sp_vr_is_used(vr)) 950 return false; 951 if (fib->lpm_tree->id == tree_id) 952 return true; 953 return false; 954 } 955 956 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 957 struct mlxsw_sp_fib *fib, 958 struct mlxsw_sp_lpm_tree *new_tree) 959 { 960 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 961 int err; 962 963 fib->lpm_tree = new_tree; 964 mlxsw_sp_lpm_tree_hold(new_tree); 965 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 966 if (err) 967 goto err_tree_bind; 968 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 969 return 0; 970 971 err_tree_bind: 972 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 973 fib->lpm_tree = old_tree; 974 return err; 975 } 976 977 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 978 struct mlxsw_sp_fib *fib, 979 struct mlxsw_sp_lpm_tree *new_tree) 980 { 981 enum mlxsw_sp_l3proto proto = fib->proto; 982 struct mlxsw_sp_lpm_tree *old_tree; 983 u8 old_id, new_id = new_tree->id; 984 struct mlxsw_sp_vr *vr; 985 int i, err; 986 987 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 988 old_id = old_tree->id; 989 990 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 991 vr = &mlxsw_sp->router->vrs[i]; 992 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 993 continue; 994 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 995 mlxsw_sp_vr_fib(vr, proto), 996 new_tree); 997 if (err) 998 goto err_tree_replace; 999 } 1000 1001 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1002 sizeof(new_tree->prefix_ref_count)); 1003 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1004 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1005 1006 return 0; 1007 1008 err_tree_replace: 1009 for (i--; i >= 0; i--) { 1010 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1011 continue; 1012 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1013 mlxsw_sp_vr_fib(vr, proto), 1014 old_tree); 1015 } 1016 return err; 1017 } 1018 1019 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1020 { 1021 struct mlxsw_sp_vr *vr; 1022 u64 max_vrs; 1023 int i; 1024 1025 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1026 return -EIO; 1027 1028 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1029 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1030 GFP_KERNEL); 1031 if (!mlxsw_sp->router->vrs) 1032 return -ENOMEM; 1033 1034 for (i = 0; i < max_vrs; i++) { 1035 vr = &mlxsw_sp->router->vrs[i]; 1036 vr->id = i; 1037 } 1038 1039 return 0; 1040 } 1041 1042 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1043 1044 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1045 { 1046 /* At this stage we're guaranteed not to have new incoming 1047 * FIB notifications and the work queue is free from FIBs 1048 * sitting on top of mlxsw netdevs. However, we can still 1049 * have other FIBs queued. Flush the queue before flushing 1050 * the device's tables. No need for locks, as we're the only 1051 * writer. 1052 */ 1053 mlxsw_core_flush_owq(); 1054 mlxsw_sp_router_fib_flush(mlxsw_sp); 1055 kfree(mlxsw_sp->router->vrs); 1056 } 1057 1058 static struct net_device * 1059 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 1060 { 1061 struct ip_tunnel *tun = netdev_priv(ol_dev); 1062 struct net *net = dev_net(ol_dev); 1063 1064 return dev_get_by_index_rcu(net, tun->parms.link); 1065 } 1066 1067 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1068 { 1069 struct net_device *d; 1070 u32 tb_id; 1071 1072 rcu_read_lock(); 1073 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1074 if (d) 1075 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1076 else 1077 tb_id = RT_TABLE_MAIN; 1078 rcu_read_unlock(); 1079 1080 return tb_id; 1081 } 1082 1083 static struct mlxsw_sp_rif * 1084 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1085 const struct mlxsw_sp_rif_params *params, 1086 struct netlink_ext_ack *extack); 1087 1088 static struct mlxsw_sp_rif_ipip_lb * 1089 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1090 enum mlxsw_sp_ipip_type ipipt, 1091 struct net_device *ol_dev, 1092 struct netlink_ext_ack *extack) 1093 { 1094 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1095 const struct mlxsw_sp_ipip_ops *ipip_ops; 1096 struct mlxsw_sp_rif *rif; 1097 1098 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1099 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1100 .common.dev = ol_dev, 1101 .common.lag = false, 1102 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1103 }; 1104 1105 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1106 if (IS_ERR(rif)) 1107 return ERR_CAST(rif); 1108 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1109 } 1110 1111 static struct mlxsw_sp_ipip_entry * 1112 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1113 enum mlxsw_sp_ipip_type ipipt, 1114 struct net_device *ol_dev) 1115 { 1116 const struct mlxsw_sp_ipip_ops *ipip_ops; 1117 struct mlxsw_sp_ipip_entry *ipip_entry; 1118 struct mlxsw_sp_ipip_entry *ret = NULL; 1119 1120 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1121 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1122 if (!ipip_entry) 1123 return ERR_PTR(-ENOMEM); 1124 1125 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1126 ol_dev, NULL); 1127 if (IS_ERR(ipip_entry->ol_lb)) { 1128 ret = ERR_CAST(ipip_entry->ol_lb); 1129 goto err_ol_ipip_lb_create; 1130 } 1131 1132 ipip_entry->ipipt = ipipt; 1133 ipip_entry->ol_dev = ol_dev; 1134 1135 switch (ipip_ops->ul_proto) { 1136 case MLXSW_SP_L3_PROTO_IPV4: 1137 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1138 break; 1139 case MLXSW_SP_L3_PROTO_IPV6: 1140 WARN_ON(1); 1141 break; 1142 } 1143 1144 return ipip_entry; 1145 1146 err_ol_ipip_lb_create: 1147 kfree(ipip_entry); 1148 return ret; 1149 } 1150 1151 static void 1152 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1153 { 1154 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1155 kfree(ipip_entry); 1156 } 1157 1158 static bool 1159 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1160 const enum mlxsw_sp_l3proto ul_proto, 1161 union mlxsw_sp_l3addr saddr, 1162 u32 ul_tb_id, 1163 struct mlxsw_sp_ipip_entry *ipip_entry) 1164 { 1165 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1166 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1167 union mlxsw_sp_l3addr tun_saddr; 1168 1169 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1170 return false; 1171 1172 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1173 return tun_ul_tb_id == ul_tb_id && 1174 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1175 } 1176 1177 static int 1178 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1179 struct mlxsw_sp_fib_entry *fib_entry, 1180 struct mlxsw_sp_ipip_entry *ipip_entry) 1181 { 1182 u32 tunnel_index; 1183 int err; 1184 1185 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1186 1, &tunnel_index); 1187 if (err) 1188 return err; 1189 1190 ipip_entry->decap_fib_entry = fib_entry; 1191 fib_entry->decap.ipip_entry = ipip_entry; 1192 fib_entry->decap.tunnel_index = tunnel_index; 1193 return 0; 1194 } 1195 1196 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1197 struct mlxsw_sp_fib_entry *fib_entry) 1198 { 1199 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1200 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1201 fib_entry->decap.ipip_entry = NULL; 1202 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1203 1, fib_entry->decap.tunnel_index); 1204 } 1205 1206 static struct mlxsw_sp_fib_node * 1207 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1208 size_t addr_len, unsigned char prefix_len); 1209 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_fib_entry *fib_entry); 1211 1212 static void 1213 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1214 struct mlxsw_sp_ipip_entry *ipip_entry) 1215 { 1216 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1217 1218 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1219 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1220 1221 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1222 } 1223 1224 static void 1225 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1226 struct mlxsw_sp_ipip_entry *ipip_entry, 1227 struct mlxsw_sp_fib_entry *decap_fib_entry) 1228 { 1229 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1230 ipip_entry)) 1231 return; 1232 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1233 1234 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1235 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1236 } 1237 1238 static struct mlxsw_sp_fib_entry * 1239 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1240 enum mlxsw_sp_l3proto proto, 1241 const union mlxsw_sp_l3addr *addr, 1242 enum mlxsw_sp_fib_entry_type type) 1243 { 1244 struct mlxsw_sp_fib_node *fib_node; 1245 unsigned char addr_prefix_len; 1246 struct mlxsw_sp_fib *fib; 1247 struct mlxsw_sp_vr *vr; 1248 const void *addrp; 1249 size_t addr_len; 1250 u32 addr4; 1251 1252 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1253 if (!vr) 1254 return NULL; 1255 fib = mlxsw_sp_vr_fib(vr, proto); 1256 1257 switch (proto) { 1258 case MLXSW_SP_L3_PROTO_IPV4: 1259 addr4 = be32_to_cpu(addr->addr4); 1260 addrp = &addr4; 1261 addr_len = 4; 1262 addr_prefix_len = 32; 1263 break; 1264 case MLXSW_SP_L3_PROTO_IPV6: 1265 default: 1266 WARN_ON(1); 1267 return NULL; 1268 } 1269 1270 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1271 addr_prefix_len); 1272 if (!fib_node || fib_node->fib_entry->type != type) 1273 return NULL; 1274 1275 return fib_node->fib_entry; 1276 } 1277 1278 /* Given an IPIP entry, find the corresponding decap route. */ 1279 static struct mlxsw_sp_fib_entry * 1280 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1281 struct mlxsw_sp_ipip_entry *ipip_entry) 1282 { 1283 static struct mlxsw_sp_fib_node *fib_node; 1284 const struct mlxsw_sp_ipip_ops *ipip_ops; 1285 unsigned char saddr_prefix_len; 1286 union mlxsw_sp_l3addr saddr; 1287 struct mlxsw_sp_fib *ul_fib; 1288 struct mlxsw_sp_vr *ul_vr; 1289 const void *saddrp; 1290 size_t saddr_len; 1291 u32 ul_tb_id; 1292 u32 saddr4; 1293 1294 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1295 1296 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1297 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1298 if (!ul_vr) 1299 return NULL; 1300 1301 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1302 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1303 ipip_entry->ol_dev); 1304 1305 switch (ipip_ops->ul_proto) { 1306 case MLXSW_SP_L3_PROTO_IPV4: 1307 saddr4 = be32_to_cpu(saddr.addr4); 1308 saddrp = &saddr4; 1309 saddr_len = 4; 1310 saddr_prefix_len = 32; 1311 break; 1312 default: 1313 WARN_ON(1); 1314 return NULL; 1315 } 1316 1317 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1318 saddr_prefix_len); 1319 if (!fib_node || 1320 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1321 return NULL; 1322 1323 return fib_node->fib_entry; 1324 } 1325 1326 static struct mlxsw_sp_ipip_entry * 1327 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1328 enum mlxsw_sp_ipip_type ipipt, 1329 struct net_device *ol_dev) 1330 { 1331 struct mlxsw_sp_ipip_entry *ipip_entry; 1332 1333 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1334 if (IS_ERR(ipip_entry)) 1335 return ipip_entry; 1336 1337 list_add_tail(&ipip_entry->ipip_list_node, 1338 &mlxsw_sp->router->ipip_list); 1339 1340 return ipip_entry; 1341 } 1342 1343 static void 1344 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1345 struct mlxsw_sp_ipip_entry *ipip_entry) 1346 { 1347 list_del(&ipip_entry->ipip_list_node); 1348 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1349 } 1350 1351 static bool 1352 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1353 const struct net_device *ul_dev, 1354 enum mlxsw_sp_l3proto ul_proto, 1355 union mlxsw_sp_l3addr ul_dip, 1356 struct mlxsw_sp_ipip_entry *ipip_entry) 1357 { 1358 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1359 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1360 1361 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1362 return false; 1363 1364 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1365 ul_tb_id, ipip_entry); 1366 } 1367 1368 /* Given decap parameters, find the corresponding IPIP entry. */ 1369 static struct mlxsw_sp_ipip_entry * 1370 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1371 enum mlxsw_sp_l3proto ul_proto, 1372 union mlxsw_sp_l3addr ul_dip) 1373 { 1374 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1375 struct net_device *ul_dev; 1376 1377 rcu_read_lock(); 1378 1379 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1380 if (!ul_dev) 1381 goto out_unlock; 1382 1383 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1384 ipip_list_node) 1385 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1386 ul_proto, ul_dip, 1387 ipip_entry)) 1388 goto out_unlock; 1389 1390 rcu_read_unlock(); 1391 1392 return NULL; 1393 1394 out_unlock: 1395 rcu_read_unlock(); 1396 return ipip_entry; 1397 } 1398 1399 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1400 const struct net_device *dev, 1401 enum mlxsw_sp_ipip_type *p_type) 1402 { 1403 struct mlxsw_sp_router *router = mlxsw_sp->router; 1404 const struct mlxsw_sp_ipip_ops *ipip_ops; 1405 enum mlxsw_sp_ipip_type ipipt; 1406 1407 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1408 ipip_ops = router->ipip_ops_arr[ipipt]; 1409 if (dev->type == ipip_ops->dev_type) { 1410 if (p_type) 1411 *p_type = ipipt; 1412 return true; 1413 } 1414 } 1415 return false; 1416 } 1417 1418 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1419 const struct net_device *dev) 1420 { 1421 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1422 } 1423 1424 static struct mlxsw_sp_ipip_entry * 1425 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1426 const struct net_device *ol_dev) 1427 { 1428 struct mlxsw_sp_ipip_entry *ipip_entry; 1429 1430 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1431 ipip_list_node) 1432 if (ipip_entry->ol_dev == ol_dev) 1433 return ipip_entry; 1434 1435 return NULL; 1436 } 1437 1438 static struct mlxsw_sp_ipip_entry * 1439 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1440 const struct net_device *ul_dev, 1441 struct mlxsw_sp_ipip_entry *start) 1442 { 1443 struct mlxsw_sp_ipip_entry *ipip_entry; 1444 1445 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1446 ipip_list_node); 1447 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1448 ipip_list_node) { 1449 struct net_device *ol_dev = ipip_entry->ol_dev; 1450 struct net_device *ipip_ul_dev; 1451 1452 rcu_read_lock(); 1453 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1454 rcu_read_unlock(); 1455 1456 if (ipip_ul_dev == ul_dev) 1457 return ipip_entry; 1458 } 1459 1460 return NULL; 1461 } 1462 1463 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1464 const struct net_device *dev) 1465 { 1466 bool is_ipip_ul; 1467 1468 mutex_lock(&mlxsw_sp->router->lock); 1469 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1470 mutex_unlock(&mlxsw_sp->router->lock); 1471 1472 return is_ipip_ul; 1473 } 1474 1475 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1476 const struct net_device *ol_dev, 1477 enum mlxsw_sp_ipip_type ipipt) 1478 { 1479 const struct mlxsw_sp_ipip_ops *ops 1480 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1481 1482 return ops->can_offload(mlxsw_sp, ol_dev); 1483 } 1484 1485 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1486 struct net_device *ol_dev) 1487 { 1488 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1489 struct mlxsw_sp_ipip_entry *ipip_entry; 1490 enum mlxsw_sp_l3proto ul_proto; 1491 union mlxsw_sp_l3addr saddr; 1492 u32 ul_tb_id; 1493 1494 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1495 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1496 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1497 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1498 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1499 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1500 saddr, ul_tb_id, 1501 NULL)) { 1502 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1503 ol_dev); 1504 if (IS_ERR(ipip_entry)) 1505 return PTR_ERR(ipip_entry); 1506 } 1507 } 1508 1509 return 0; 1510 } 1511 1512 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1513 struct net_device *ol_dev) 1514 { 1515 struct mlxsw_sp_ipip_entry *ipip_entry; 1516 1517 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1518 if (ipip_entry) 1519 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1520 } 1521 1522 static void 1523 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1524 struct mlxsw_sp_ipip_entry *ipip_entry) 1525 { 1526 struct mlxsw_sp_fib_entry *decap_fib_entry; 1527 1528 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1529 if (decap_fib_entry) 1530 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1531 decap_fib_entry); 1532 } 1533 1534 static int 1535 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1536 u16 ul_rif_id, bool enable) 1537 { 1538 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1539 struct mlxsw_sp_rif *rif = &lb_rif->common; 1540 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1541 char ritr_pl[MLXSW_REG_RITR_LEN]; 1542 u32 saddr4; 1543 1544 switch (lb_cf.ul_protocol) { 1545 case MLXSW_SP_L3_PROTO_IPV4: 1546 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1547 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1548 rif->rif_index, rif->vr_id, rif->dev->mtu); 1549 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1550 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1551 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1552 break; 1553 1554 case MLXSW_SP_L3_PROTO_IPV6: 1555 return -EAFNOSUPPORT; 1556 } 1557 1558 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1559 } 1560 1561 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1562 struct net_device *ol_dev) 1563 { 1564 struct mlxsw_sp_ipip_entry *ipip_entry; 1565 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1566 int err = 0; 1567 1568 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1569 if (ipip_entry) { 1570 lb_rif = ipip_entry->ol_lb; 1571 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1572 lb_rif->ul_rif_id, true); 1573 if (err) 1574 goto out; 1575 lb_rif->common.mtu = ol_dev->mtu; 1576 } 1577 1578 out: 1579 return err; 1580 } 1581 1582 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1583 struct net_device *ol_dev) 1584 { 1585 struct mlxsw_sp_ipip_entry *ipip_entry; 1586 1587 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1588 if (ipip_entry) 1589 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1590 } 1591 1592 static void 1593 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1594 struct mlxsw_sp_ipip_entry *ipip_entry) 1595 { 1596 if (ipip_entry->decap_fib_entry) 1597 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1598 } 1599 1600 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1601 struct net_device *ol_dev) 1602 { 1603 struct mlxsw_sp_ipip_entry *ipip_entry; 1604 1605 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1606 if (ipip_entry) 1607 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1608 } 1609 1610 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_rif *old_rif, 1612 struct mlxsw_sp_rif *new_rif); 1613 static int 1614 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1615 struct mlxsw_sp_ipip_entry *ipip_entry, 1616 bool keep_encap, 1617 struct netlink_ext_ack *extack) 1618 { 1619 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1620 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1621 1622 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1623 ipip_entry->ipipt, 1624 ipip_entry->ol_dev, 1625 extack); 1626 if (IS_ERR(new_lb_rif)) 1627 return PTR_ERR(new_lb_rif); 1628 ipip_entry->ol_lb = new_lb_rif; 1629 1630 if (keep_encap) 1631 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1632 &new_lb_rif->common); 1633 1634 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1635 1636 return 0; 1637 } 1638 1639 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1640 struct mlxsw_sp_rif *rif); 1641 1642 /** 1643 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1644 * @mlxsw_sp: mlxsw_sp. 1645 * @ipip_entry: IPIP entry. 1646 * @recreate_loopback: Recreates the associated loopback RIF. 1647 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1648 * relevant when recreate_loopback is true. 1649 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1650 * is only relevant when recreate_loopback is false. 1651 * @extack: extack. 1652 * 1653 * Return: Non-zero value on failure. 1654 */ 1655 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1656 struct mlxsw_sp_ipip_entry *ipip_entry, 1657 bool recreate_loopback, 1658 bool keep_encap, 1659 bool update_nexthops, 1660 struct netlink_ext_ack *extack) 1661 { 1662 int err; 1663 1664 /* RIFs can't be edited, so to update loopback, we need to destroy and 1665 * recreate it. That creates a window of opportunity where RALUE and 1666 * RATR registers end up referencing a RIF that's already gone. RATRs 1667 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1668 * of RALUE, demote the decap route back. 1669 */ 1670 if (ipip_entry->decap_fib_entry) 1671 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1672 1673 if (recreate_loopback) { 1674 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1675 keep_encap, extack); 1676 if (err) 1677 return err; 1678 } else if (update_nexthops) { 1679 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1680 &ipip_entry->ol_lb->common); 1681 } 1682 1683 if (ipip_entry->ol_dev->flags & IFF_UP) 1684 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1685 1686 return 0; 1687 } 1688 1689 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1690 struct net_device *ol_dev, 1691 struct netlink_ext_ack *extack) 1692 { 1693 struct mlxsw_sp_ipip_entry *ipip_entry = 1694 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1695 1696 if (!ipip_entry) 1697 return 0; 1698 1699 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1700 true, false, false, extack); 1701 } 1702 1703 static int 1704 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1705 struct mlxsw_sp_ipip_entry *ipip_entry, 1706 struct net_device *ul_dev, 1707 bool *demote_this, 1708 struct netlink_ext_ack *extack) 1709 { 1710 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1711 enum mlxsw_sp_l3proto ul_proto; 1712 union mlxsw_sp_l3addr saddr; 1713 1714 /* Moving underlay to a different VRF might cause local address 1715 * conflict, and the conflicting tunnels need to be demoted. 1716 */ 1717 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1718 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1719 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1720 saddr, ul_tb_id, 1721 ipip_entry)) { 1722 *demote_this = true; 1723 return 0; 1724 } 1725 1726 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1727 true, true, false, extack); 1728 } 1729 1730 static int 1731 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1732 struct mlxsw_sp_ipip_entry *ipip_entry, 1733 struct net_device *ul_dev) 1734 { 1735 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1736 false, false, true, NULL); 1737 } 1738 1739 static int 1740 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1741 struct mlxsw_sp_ipip_entry *ipip_entry, 1742 struct net_device *ul_dev) 1743 { 1744 /* A down underlay device causes encapsulated packets to not be 1745 * forwarded, but decap still works. So refresh next hops without 1746 * touching anything else. 1747 */ 1748 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1749 false, false, true, NULL); 1750 } 1751 1752 static int 1753 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1754 struct net_device *ol_dev, 1755 struct netlink_ext_ack *extack) 1756 { 1757 const struct mlxsw_sp_ipip_ops *ipip_ops; 1758 struct mlxsw_sp_ipip_entry *ipip_entry; 1759 int err; 1760 1761 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1762 if (!ipip_entry) 1763 /* A change might make a tunnel eligible for offloading, but 1764 * that is currently not implemented. What falls to slow path 1765 * stays there. 1766 */ 1767 return 0; 1768 1769 /* A change might make a tunnel not eligible for offloading. */ 1770 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1771 ipip_entry->ipipt)) { 1772 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1773 return 0; 1774 } 1775 1776 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1777 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1778 return err; 1779 } 1780 1781 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1782 struct mlxsw_sp_ipip_entry *ipip_entry) 1783 { 1784 struct net_device *ol_dev = ipip_entry->ol_dev; 1785 1786 if (ol_dev->flags & IFF_UP) 1787 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1788 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1789 } 1790 1791 /* The configuration where several tunnels have the same local address in the 1792 * same underlay table needs special treatment in the HW. That is currently not 1793 * implemented in the driver. This function finds and demotes the first tunnel 1794 * with a given source address, except the one passed in in the argument 1795 * `except'. 1796 */ 1797 bool 1798 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1799 enum mlxsw_sp_l3proto ul_proto, 1800 union mlxsw_sp_l3addr saddr, 1801 u32 ul_tb_id, 1802 const struct mlxsw_sp_ipip_entry *except) 1803 { 1804 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1805 1806 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1807 ipip_list_node) { 1808 if (ipip_entry != except && 1809 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1810 ul_tb_id, ipip_entry)) { 1811 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1812 return true; 1813 } 1814 } 1815 1816 return false; 1817 } 1818 1819 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1820 struct net_device *ul_dev) 1821 { 1822 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1823 1824 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1825 ipip_list_node) { 1826 struct net_device *ol_dev = ipip_entry->ol_dev; 1827 struct net_device *ipip_ul_dev; 1828 1829 rcu_read_lock(); 1830 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1831 rcu_read_unlock(); 1832 if (ipip_ul_dev == ul_dev) 1833 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1834 } 1835 } 1836 1837 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1838 struct net_device *ol_dev, 1839 unsigned long event, 1840 struct netdev_notifier_info *info) 1841 { 1842 struct netdev_notifier_changeupper_info *chup; 1843 struct netlink_ext_ack *extack; 1844 int err = 0; 1845 1846 mutex_lock(&mlxsw_sp->router->lock); 1847 switch (event) { 1848 case NETDEV_REGISTER: 1849 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1850 break; 1851 case NETDEV_UNREGISTER: 1852 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1853 break; 1854 case NETDEV_UP: 1855 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1856 break; 1857 case NETDEV_DOWN: 1858 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1859 break; 1860 case NETDEV_CHANGEUPPER: 1861 chup = container_of(info, typeof(*chup), info); 1862 extack = info->extack; 1863 if (netif_is_l3_master(chup->upper_dev)) 1864 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1865 ol_dev, 1866 extack); 1867 break; 1868 case NETDEV_CHANGE: 1869 extack = info->extack; 1870 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1871 ol_dev, extack); 1872 break; 1873 case NETDEV_CHANGEMTU: 1874 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1875 break; 1876 } 1877 mutex_unlock(&mlxsw_sp->router->lock); 1878 return err; 1879 } 1880 1881 static int 1882 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1883 struct mlxsw_sp_ipip_entry *ipip_entry, 1884 struct net_device *ul_dev, 1885 bool *demote_this, 1886 unsigned long event, 1887 struct netdev_notifier_info *info) 1888 { 1889 struct netdev_notifier_changeupper_info *chup; 1890 struct netlink_ext_ack *extack; 1891 1892 switch (event) { 1893 case NETDEV_CHANGEUPPER: 1894 chup = container_of(info, typeof(*chup), info); 1895 extack = info->extack; 1896 if (netif_is_l3_master(chup->upper_dev)) 1897 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1898 ipip_entry, 1899 ul_dev, 1900 demote_this, 1901 extack); 1902 break; 1903 1904 case NETDEV_UP: 1905 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1906 ul_dev); 1907 case NETDEV_DOWN: 1908 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1909 ipip_entry, 1910 ul_dev); 1911 } 1912 return 0; 1913 } 1914 1915 int 1916 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1917 struct net_device *ul_dev, 1918 unsigned long event, 1919 struct netdev_notifier_info *info) 1920 { 1921 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1922 int err = 0; 1923 1924 mutex_lock(&mlxsw_sp->router->lock); 1925 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1926 ul_dev, 1927 ipip_entry))) { 1928 struct mlxsw_sp_ipip_entry *prev; 1929 bool demote_this = false; 1930 1931 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1932 ul_dev, &demote_this, 1933 event, info); 1934 if (err) { 1935 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1936 ul_dev); 1937 break; 1938 } 1939 1940 if (demote_this) { 1941 if (list_is_first(&ipip_entry->ipip_list_node, 1942 &mlxsw_sp->router->ipip_list)) 1943 prev = NULL; 1944 else 1945 /* This can't be cached from previous iteration, 1946 * because that entry could be gone now. 1947 */ 1948 prev = list_prev_entry(ipip_entry, 1949 ipip_list_node); 1950 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1951 ipip_entry = prev; 1952 } 1953 } 1954 mutex_unlock(&mlxsw_sp->router->lock); 1955 1956 return err; 1957 } 1958 1959 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1960 enum mlxsw_sp_l3proto ul_proto, 1961 const union mlxsw_sp_l3addr *ul_sip, 1962 u32 tunnel_index) 1963 { 1964 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1965 struct mlxsw_sp_router *router = mlxsw_sp->router; 1966 struct mlxsw_sp_fib_entry *fib_entry; 1967 int err = 0; 1968 1969 mutex_lock(&mlxsw_sp->router->lock); 1970 1971 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1972 err = -EINVAL; 1973 goto out; 1974 } 1975 1976 router->nve_decap_config.ul_tb_id = ul_tb_id; 1977 router->nve_decap_config.tunnel_index = tunnel_index; 1978 router->nve_decap_config.ul_proto = ul_proto; 1979 router->nve_decap_config.ul_sip = *ul_sip; 1980 router->nve_decap_config.valid = true; 1981 1982 /* It is valid to create a tunnel with a local IP and only later 1983 * assign this IP address to a local interface 1984 */ 1985 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1986 ul_proto, ul_sip, 1987 type); 1988 if (!fib_entry) 1989 goto out; 1990 1991 fib_entry->decap.tunnel_index = tunnel_index; 1992 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1993 1994 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1995 if (err) 1996 goto err_fib_entry_update; 1997 1998 goto out; 1999 2000 err_fib_entry_update: 2001 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2002 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2003 out: 2004 mutex_unlock(&mlxsw_sp->router->lock); 2005 return err; 2006 } 2007 2008 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2009 enum mlxsw_sp_l3proto ul_proto, 2010 const union mlxsw_sp_l3addr *ul_sip) 2011 { 2012 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2013 struct mlxsw_sp_router *router = mlxsw_sp->router; 2014 struct mlxsw_sp_fib_entry *fib_entry; 2015 2016 mutex_lock(&mlxsw_sp->router->lock); 2017 2018 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2019 goto out; 2020 2021 router->nve_decap_config.valid = false; 2022 2023 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2024 ul_proto, ul_sip, 2025 type); 2026 if (!fib_entry) 2027 goto out; 2028 2029 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2030 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2031 out: 2032 mutex_unlock(&mlxsw_sp->router->lock); 2033 } 2034 2035 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2036 u32 ul_tb_id, 2037 enum mlxsw_sp_l3proto ul_proto, 2038 const union mlxsw_sp_l3addr *ul_sip) 2039 { 2040 struct mlxsw_sp_router *router = mlxsw_sp->router; 2041 2042 return router->nve_decap_config.valid && 2043 router->nve_decap_config.ul_tb_id == ul_tb_id && 2044 router->nve_decap_config.ul_proto == ul_proto && 2045 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2046 sizeof(*ul_sip)); 2047 } 2048 2049 struct mlxsw_sp_neigh_key { 2050 struct neighbour *n; 2051 }; 2052 2053 struct mlxsw_sp_neigh_entry { 2054 struct list_head rif_list_node; 2055 struct rhash_head ht_node; 2056 struct mlxsw_sp_neigh_key key; 2057 u16 rif; 2058 bool connected; 2059 unsigned char ha[ETH_ALEN]; 2060 struct list_head nexthop_list; /* list of nexthops using 2061 * this neigh entry 2062 */ 2063 struct list_head nexthop_neighs_list_node; 2064 unsigned int counter_index; 2065 bool counter_valid; 2066 }; 2067 2068 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2069 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2070 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2071 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2072 }; 2073 2074 struct mlxsw_sp_neigh_entry * 2075 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2076 struct mlxsw_sp_neigh_entry *neigh_entry) 2077 { 2078 if (!neigh_entry) { 2079 if (list_empty(&rif->neigh_list)) 2080 return NULL; 2081 else 2082 return list_first_entry(&rif->neigh_list, 2083 typeof(*neigh_entry), 2084 rif_list_node); 2085 } 2086 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2087 return NULL; 2088 return list_next_entry(neigh_entry, rif_list_node); 2089 } 2090 2091 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2092 { 2093 return neigh_entry->key.n->tbl->family; 2094 } 2095 2096 unsigned char * 2097 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2098 { 2099 return neigh_entry->ha; 2100 } 2101 2102 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2103 { 2104 struct neighbour *n; 2105 2106 n = neigh_entry->key.n; 2107 return ntohl(*((__be32 *) n->primary_key)); 2108 } 2109 2110 struct in6_addr * 2111 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2112 { 2113 struct neighbour *n; 2114 2115 n = neigh_entry->key.n; 2116 return (struct in6_addr *) &n->primary_key; 2117 } 2118 2119 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2120 struct mlxsw_sp_neigh_entry *neigh_entry, 2121 u64 *p_counter) 2122 { 2123 if (!neigh_entry->counter_valid) 2124 return -EINVAL; 2125 2126 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2127 p_counter, NULL); 2128 } 2129 2130 static struct mlxsw_sp_neigh_entry * 2131 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2132 u16 rif) 2133 { 2134 struct mlxsw_sp_neigh_entry *neigh_entry; 2135 2136 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2137 if (!neigh_entry) 2138 return NULL; 2139 2140 neigh_entry->key.n = n; 2141 neigh_entry->rif = rif; 2142 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2143 2144 return neigh_entry; 2145 } 2146 2147 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2148 { 2149 kfree(neigh_entry); 2150 } 2151 2152 static int 2153 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2154 struct mlxsw_sp_neigh_entry *neigh_entry) 2155 { 2156 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2157 &neigh_entry->ht_node, 2158 mlxsw_sp_neigh_ht_params); 2159 } 2160 2161 static void 2162 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2163 struct mlxsw_sp_neigh_entry *neigh_entry) 2164 { 2165 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2166 &neigh_entry->ht_node, 2167 mlxsw_sp_neigh_ht_params); 2168 } 2169 2170 static bool 2171 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2172 struct mlxsw_sp_neigh_entry *neigh_entry) 2173 { 2174 struct devlink *devlink; 2175 const char *table_name; 2176 2177 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2178 case AF_INET: 2179 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2180 break; 2181 case AF_INET6: 2182 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2183 break; 2184 default: 2185 WARN_ON(1); 2186 return false; 2187 } 2188 2189 devlink = priv_to_devlink(mlxsw_sp->core); 2190 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2191 } 2192 2193 static void 2194 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2195 struct mlxsw_sp_neigh_entry *neigh_entry) 2196 { 2197 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2198 return; 2199 2200 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2201 return; 2202 2203 neigh_entry->counter_valid = true; 2204 } 2205 2206 static void 2207 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2208 struct mlxsw_sp_neigh_entry *neigh_entry) 2209 { 2210 if (!neigh_entry->counter_valid) 2211 return; 2212 mlxsw_sp_flow_counter_free(mlxsw_sp, 2213 neigh_entry->counter_index); 2214 neigh_entry->counter_valid = false; 2215 } 2216 2217 static struct mlxsw_sp_neigh_entry * 2218 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2219 { 2220 struct mlxsw_sp_neigh_entry *neigh_entry; 2221 struct mlxsw_sp_rif *rif; 2222 int err; 2223 2224 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2225 if (!rif) 2226 return ERR_PTR(-EINVAL); 2227 2228 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2229 if (!neigh_entry) 2230 return ERR_PTR(-ENOMEM); 2231 2232 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2233 if (err) 2234 goto err_neigh_entry_insert; 2235 2236 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2237 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2238 2239 return neigh_entry; 2240 2241 err_neigh_entry_insert: 2242 mlxsw_sp_neigh_entry_free(neigh_entry); 2243 return ERR_PTR(err); 2244 } 2245 2246 static void 2247 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2248 struct mlxsw_sp_neigh_entry *neigh_entry) 2249 { 2250 list_del(&neigh_entry->rif_list_node); 2251 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2252 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2253 mlxsw_sp_neigh_entry_free(neigh_entry); 2254 } 2255 2256 static struct mlxsw_sp_neigh_entry * 2257 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2258 { 2259 struct mlxsw_sp_neigh_key key; 2260 2261 key.n = n; 2262 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2263 &key, mlxsw_sp_neigh_ht_params); 2264 } 2265 2266 static void 2267 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2268 { 2269 unsigned long interval; 2270 2271 #if IS_ENABLED(CONFIG_IPV6) 2272 interval = min_t(unsigned long, 2273 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2274 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2275 #else 2276 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2277 #endif 2278 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2279 } 2280 2281 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2282 char *rauhtd_pl, 2283 int ent_index) 2284 { 2285 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2286 struct net_device *dev; 2287 struct neighbour *n; 2288 __be32 dipn; 2289 u32 dip; 2290 u16 rif; 2291 2292 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2293 2294 if (WARN_ON_ONCE(rif >= max_rifs)) 2295 return; 2296 if (!mlxsw_sp->router->rifs[rif]) { 2297 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2298 return; 2299 } 2300 2301 dipn = htonl(dip); 2302 dev = mlxsw_sp->router->rifs[rif]->dev; 2303 n = neigh_lookup(&arp_tbl, &dipn, dev); 2304 if (!n) 2305 return; 2306 2307 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2308 neigh_event_send(n, NULL); 2309 neigh_release(n); 2310 } 2311 2312 #if IS_ENABLED(CONFIG_IPV6) 2313 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2314 char *rauhtd_pl, 2315 int rec_index) 2316 { 2317 struct net_device *dev; 2318 struct neighbour *n; 2319 struct in6_addr dip; 2320 u16 rif; 2321 2322 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2323 (char *) &dip); 2324 2325 if (!mlxsw_sp->router->rifs[rif]) { 2326 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2327 return; 2328 } 2329 2330 dev = mlxsw_sp->router->rifs[rif]->dev; 2331 n = neigh_lookup(&nd_tbl, &dip, dev); 2332 if (!n) 2333 return; 2334 2335 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2336 neigh_event_send(n, NULL); 2337 neigh_release(n); 2338 } 2339 #else 2340 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2341 char *rauhtd_pl, 2342 int rec_index) 2343 { 2344 } 2345 #endif 2346 2347 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2348 char *rauhtd_pl, 2349 int rec_index) 2350 { 2351 u8 num_entries; 2352 int i; 2353 2354 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2355 rec_index); 2356 /* Hardware starts counting at 0, so add 1. */ 2357 num_entries++; 2358 2359 /* Each record consists of several neighbour entries. */ 2360 for (i = 0; i < num_entries; i++) { 2361 int ent_index; 2362 2363 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2364 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2365 ent_index); 2366 } 2367 2368 } 2369 2370 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2371 char *rauhtd_pl, 2372 int rec_index) 2373 { 2374 /* One record contains one entry. */ 2375 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2376 rec_index); 2377 } 2378 2379 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2380 char *rauhtd_pl, int rec_index) 2381 { 2382 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2383 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2384 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2385 rec_index); 2386 break; 2387 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2388 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2389 rec_index); 2390 break; 2391 } 2392 } 2393 2394 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2395 { 2396 u8 num_rec, last_rec_index, num_entries; 2397 2398 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2399 last_rec_index = num_rec - 1; 2400 2401 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2402 return false; 2403 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2404 MLXSW_REG_RAUHTD_TYPE_IPV6) 2405 return true; 2406 2407 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2408 last_rec_index); 2409 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2410 return true; 2411 return false; 2412 } 2413 2414 static int 2415 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2416 char *rauhtd_pl, 2417 enum mlxsw_reg_rauhtd_type type) 2418 { 2419 int i, num_rec; 2420 int err; 2421 2422 /* Ensure the RIF we read from the device does not change mid-dump. */ 2423 mutex_lock(&mlxsw_sp->router->lock); 2424 do { 2425 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2426 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2427 rauhtd_pl); 2428 if (err) { 2429 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2430 break; 2431 } 2432 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2433 for (i = 0; i < num_rec; i++) 2434 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2435 i); 2436 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2437 mutex_unlock(&mlxsw_sp->router->lock); 2438 2439 return err; 2440 } 2441 2442 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2443 { 2444 enum mlxsw_reg_rauhtd_type type; 2445 char *rauhtd_pl; 2446 int err; 2447 2448 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2449 if (!rauhtd_pl) 2450 return -ENOMEM; 2451 2452 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2453 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2454 if (err) 2455 goto out; 2456 2457 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2458 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2459 out: 2460 kfree(rauhtd_pl); 2461 return err; 2462 } 2463 2464 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2465 { 2466 struct mlxsw_sp_neigh_entry *neigh_entry; 2467 2468 mutex_lock(&mlxsw_sp->router->lock); 2469 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2470 nexthop_neighs_list_node) 2471 /* If this neigh have nexthops, make the kernel think this neigh 2472 * is active regardless of the traffic. 2473 */ 2474 neigh_event_send(neigh_entry->key.n, NULL); 2475 mutex_unlock(&mlxsw_sp->router->lock); 2476 } 2477 2478 static void 2479 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2480 { 2481 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2482 2483 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2484 msecs_to_jiffies(interval)); 2485 } 2486 2487 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2488 { 2489 struct mlxsw_sp_router *router; 2490 int err; 2491 2492 router = container_of(work, struct mlxsw_sp_router, 2493 neighs_update.dw.work); 2494 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2495 if (err) 2496 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2497 2498 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2499 2500 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2501 } 2502 2503 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2504 { 2505 struct mlxsw_sp_neigh_entry *neigh_entry; 2506 struct mlxsw_sp_router *router; 2507 2508 router = container_of(work, struct mlxsw_sp_router, 2509 nexthop_probe_dw.work); 2510 /* Iterate over nexthop neighbours, find those who are unresolved and 2511 * send arp on them. This solves the chicken-egg problem when 2512 * the nexthop wouldn't get offloaded until the neighbor is resolved 2513 * but it wouldn't get resolved ever in case traffic is flowing in HW 2514 * using different nexthop. 2515 */ 2516 mutex_lock(&router->lock); 2517 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2518 nexthop_neighs_list_node) 2519 if (!neigh_entry->connected) 2520 neigh_event_send(neigh_entry->key.n, NULL); 2521 mutex_unlock(&router->lock); 2522 2523 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2524 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2525 } 2526 2527 static void 2528 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2529 struct mlxsw_sp_neigh_entry *neigh_entry, 2530 bool removing, bool dead); 2531 2532 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2533 { 2534 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2535 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2536 } 2537 2538 static int 2539 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2540 struct mlxsw_sp_neigh_entry *neigh_entry, 2541 enum mlxsw_reg_rauht_op op) 2542 { 2543 struct neighbour *n = neigh_entry->key.n; 2544 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2545 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2546 2547 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2548 dip); 2549 if (neigh_entry->counter_valid) 2550 mlxsw_reg_rauht_pack_counter(rauht_pl, 2551 neigh_entry->counter_index); 2552 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2553 } 2554 2555 static int 2556 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2557 struct mlxsw_sp_neigh_entry *neigh_entry, 2558 enum mlxsw_reg_rauht_op op) 2559 { 2560 struct neighbour *n = neigh_entry->key.n; 2561 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2562 const char *dip = n->primary_key; 2563 2564 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2565 dip); 2566 if (neigh_entry->counter_valid) 2567 mlxsw_reg_rauht_pack_counter(rauht_pl, 2568 neigh_entry->counter_index); 2569 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2570 } 2571 2572 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2573 { 2574 struct neighbour *n = neigh_entry->key.n; 2575 2576 /* Packets with a link-local destination address are trapped 2577 * after LPM lookup and never reach the neighbour table, so 2578 * there is no need to program such neighbours to the device. 2579 */ 2580 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2581 IPV6_ADDR_LINKLOCAL) 2582 return true; 2583 return false; 2584 } 2585 2586 static void 2587 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2588 struct mlxsw_sp_neigh_entry *neigh_entry, 2589 bool adding) 2590 { 2591 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2592 int err; 2593 2594 if (!adding && !neigh_entry->connected) 2595 return; 2596 neigh_entry->connected = adding; 2597 if (neigh_entry->key.n->tbl->family == AF_INET) { 2598 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2599 op); 2600 if (err) 2601 return; 2602 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2603 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2604 return; 2605 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2606 op); 2607 if (err) 2608 return; 2609 } else { 2610 WARN_ON_ONCE(1); 2611 return; 2612 } 2613 2614 if (adding) 2615 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2616 else 2617 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2618 } 2619 2620 void 2621 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2622 struct mlxsw_sp_neigh_entry *neigh_entry, 2623 bool adding) 2624 { 2625 if (adding) 2626 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2627 else 2628 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2629 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2630 } 2631 2632 struct mlxsw_sp_netevent_work { 2633 struct work_struct work; 2634 struct mlxsw_sp *mlxsw_sp; 2635 struct neighbour *n; 2636 }; 2637 2638 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2639 { 2640 struct mlxsw_sp_netevent_work *net_work = 2641 container_of(work, struct mlxsw_sp_netevent_work, work); 2642 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2643 struct mlxsw_sp_neigh_entry *neigh_entry; 2644 struct neighbour *n = net_work->n; 2645 unsigned char ha[ETH_ALEN]; 2646 bool entry_connected; 2647 u8 nud_state, dead; 2648 2649 /* If these parameters are changed after we release the lock, 2650 * then we are guaranteed to receive another event letting us 2651 * know about it. 2652 */ 2653 read_lock_bh(&n->lock); 2654 memcpy(ha, n->ha, ETH_ALEN); 2655 nud_state = n->nud_state; 2656 dead = n->dead; 2657 read_unlock_bh(&n->lock); 2658 2659 mutex_lock(&mlxsw_sp->router->lock); 2660 mlxsw_sp_span_respin(mlxsw_sp); 2661 2662 entry_connected = nud_state & NUD_VALID && !dead; 2663 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2664 if (!entry_connected && !neigh_entry) 2665 goto out; 2666 if (!neigh_entry) { 2667 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2668 if (IS_ERR(neigh_entry)) 2669 goto out; 2670 } 2671 2672 if (neigh_entry->connected && entry_connected && 2673 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2674 goto out; 2675 2676 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2677 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2678 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2679 dead); 2680 2681 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2682 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2683 2684 out: 2685 mutex_unlock(&mlxsw_sp->router->lock); 2686 neigh_release(n); 2687 kfree(net_work); 2688 } 2689 2690 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2691 2692 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2693 { 2694 struct mlxsw_sp_netevent_work *net_work = 2695 container_of(work, struct mlxsw_sp_netevent_work, work); 2696 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2697 2698 mlxsw_sp_mp_hash_init(mlxsw_sp); 2699 kfree(net_work); 2700 } 2701 2702 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2703 2704 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2705 { 2706 struct mlxsw_sp_netevent_work *net_work = 2707 container_of(work, struct mlxsw_sp_netevent_work, work); 2708 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2709 2710 __mlxsw_sp_router_init(mlxsw_sp); 2711 kfree(net_work); 2712 } 2713 2714 static int mlxsw_sp_router_schedule_work(struct net *net, 2715 struct notifier_block *nb, 2716 void (*cb)(struct work_struct *)) 2717 { 2718 struct mlxsw_sp_netevent_work *net_work; 2719 struct mlxsw_sp_router *router; 2720 2721 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2722 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2723 return NOTIFY_DONE; 2724 2725 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2726 if (!net_work) 2727 return NOTIFY_BAD; 2728 2729 INIT_WORK(&net_work->work, cb); 2730 net_work->mlxsw_sp = router->mlxsw_sp; 2731 mlxsw_core_schedule_work(&net_work->work); 2732 return NOTIFY_DONE; 2733 } 2734 2735 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2736 unsigned long event, void *ptr) 2737 { 2738 struct mlxsw_sp_netevent_work *net_work; 2739 struct mlxsw_sp_port *mlxsw_sp_port; 2740 struct mlxsw_sp *mlxsw_sp; 2741 unsigned long interval; 2742 struct neigh_parms *p; 2743 struct neighbour *n; 2744 2745 switch (event) { 2746 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2747 p = ptr; 2748 2749 /* We don't care about changes in the default table. */ 2750 if (!p->dev || (p->tbl->family != AF_INET && 2751 p->tbl->family != AF_INET6)) 2752 return NOTIFY_DONE; 2753 2754 /* We are in atomic context and can't take RTNL mutex, 2755 * so use RCU variant to walk the device chain. 2756 */ 2757 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2758 if (!mlxsw_sp_port) 2759 return NOTIFY_DONE; 2760 2761 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2762 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2763 mlxsw_sp->router->neighs_update.interval = interval; 2764 2765 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2766 break; 2767 case NETEVENT_NEIGH_UPDATE: 2768 n = ptr; 2769 2770 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2771 return NOTIFY_DONE; 2772 2773 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2774 if (!mlxsw_sp_port) 2775 return NOTIFY_DONE; 2776 2777 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2778 if (!net_work) { 2779 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2780 return NOTIFY_BAD; 2781 } 2782 2783 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2784 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2785 net_work->n = n; 2786 2787 /* Take a reference to ensure the neighbour won't be 2788 * destructed until we drop the reference in delayed 2789 * work. 2790 */ 2791 neigh_clone(n); 2792 mlxsw_core_schedule_work(&net_work->work); 2793 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2794 break; 2795 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2796 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2797 return mlxsw_sp_router_schedule_work(ptr, nb, 2798 mlxsw_sp_router_mp_hash_event_work); 2799 2800 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2801 return mlxsw_sp_router_schedule_work(ptr, nb, 2802 mlxsw_sp_router_update_priority_work); 2803 } 2804 2805 return NOTIFY_DONE; 2806 } 2807 2808 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2809 { 2810 int err; 2811 2812 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2813 &mlxsw_sp_neigh_ht_params); 2814 if (err) 2815 return err; 2816 2817 /* Initialize the polling interval according to the default 2818 * table. 2819 */ 2820 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2821 2822 /* Create the delayed works for the activity_update */ 2823 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2824 mlxsw_sp_router_neighs_update_work); 2825 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2826 mlxsw_sp_router_probe_unresolved_nexthops); 2827 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2828 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2829 return 0; 2830 } 2831 2832 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2833 { 2834 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2835 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2836 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2837 } 2838 2839 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2840 struct mlxsw_sp_rif *rif) 2841 { 2842 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2843 2844 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2845 rif_list_node) { 2846 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2847 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2848 } 2849 } 2850 2851 enum mlxsw_sp_nexthop_type { 2852 MLXSW_SP_NEXTHOP_TYPE_ETH, 2853 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2854 }; 2855 2856 enum mlxsw_sp_nexthop_action { 2857 /* Nexthop forwards packets to an egress RIF */ 2858 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2859 /* Nexthop discards packets */ 2860 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2861 /* Nexthop traps packets */ 2862 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2863 }; 2864 2865 struct mlxsw_sp_nexthop_key { 2866 struct fib_nh *fib_nh; 2867 }; 2868 2869 struct mlxsw_sp_nexthop { 2870 struct list_head neigh_list_node; /* member of neigh entry list */ 2871 struct list_head rif_list_node; 2872 struct list_head router_list_node; 2873 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2874 * this nexthop belongs to 2875 */ 2876 struct rhash_head ht_node; 2877 struct neigh_table *neigh_tbl; 2878 struct mlxsw_sp_nexthop_key key; 2879 unsigned char gw_addr[sizeof(struct in6_addr)]; 2880 int ifindex; 2881 int nh_weight; 2882 int norm_nh_weight; 2883 int num_adj_entries; 2884 struct mlxsw_sp_rif *rif; 2885 u8 should_offload:1, /* set indicates this nexthop should be written 2886 * to the adjacency table. 2887 */ 2888 offloaded:1, /* set indicates this nexthop was written to the 2889 * adjacency table. 2890 */ 2891 update:1; /* set indicates this nexthop should be updated in the 2892 * adjacency table (f.e., its MAC changed). 2893 */ 2894 enum mlxsw_sp_nexthop_action action; 2895 enum mlxsw_sp_nexthop_type type; 2896 union { 2897 struct mlxsw_sp_neigh_entry *neigh_entry; 2898 struct mlxsw_sp_ipip_entry *ipip_entry; 2899 }; 2900 unsigned int counter_index; 2901 bool counter_valid; 2902 }; 2903 2904 enum mlxsw_sp_nexthop_group_type { 2905 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2906 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2907 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2908 }; 2909 2910 struct mlxsw_sp_nexthop_group_info { 2911 struct mlxsw_sp_nexthop_group *nh_grp; 2912 u32 adj_index; 2913 u16 ecmp_size; 2914 u16 count; 2915 int sum_norm_weight; 2916 u8 adj_index_valid:1, 2917 gateway:1, /* routes using the group use a gateway */ 2918 is_resilient:1; 2919 struct list_head list; /* member in nh_res_grp_list */ 2920 struct mlxsw_sp_nexthop nexthops[0]; 2921 #define nh_rif nexthops[0].rif 2922 }; 2923 2924 struct mlxsw_sp_nexthop_group_vr_key { 2925 u16 vr_id; 2926 enum mlxsw_sp_l3proto proto; 2927 }; 2928 2929 struct mlxsw_sp_nexthop_group_vr_entry { 2930 struct list_head list; /* member in vr_list */ 2931 struct rhash_head ht_node; /* member in vr_ht */ 2932 refcount_t ref_count; 2933 struct mlxsw_sp_nexthop_group_vr_key key; 2934 }; 2935 2936 struct mlxsw_sp_nexthop_group { 2937 struct rhash_head ht_node; 2938 struct list_head fib_list; /* list of fib entries that use this group */ 2939 union { 2940 struct { 2941 struct fib_info *fi; 2942 } ipv4; 2943 struct { 2944 u32 id; 2945 } obj; 2946 }; 2947 struct mlxsw_sp_nexthop_group_info *nhgi; 2948 struct list_head vr_list; 2949 struct rhashtable vr_ht; 2950 enum mlxsw_sp_nexthop_group_type type; 2951 bool can_destroy; 2952 }; 2953 2954 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2955 struct mlxsw_sp_nexthop *nh) 2956 { 2957 struct devlink *devlink; 2958 2959 devlink = priv_to_devlink(mlxsw_sp->core); 2960 if (!devlink_dpipe_table_counter_enabled(devlink, 2961 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2962 return; 2963 2964 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2965 return; 2966 2967 nh->counter_valid = true; 2968 } 2969 2970 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2971 struct mlxsw_sp_nexthop *nh) 2972 { 2973 if (!nh->counter_valid) 2974 return; 2975 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2976 nh->counter_valid = false; 2977 } 2978 2979 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2980 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2981 { 2982 if (!nh->counter_valid) 2983 return -EINVAL; 2984 2985 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2986 p_counter, NULL); 2987 } 2988 2989 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2990 struct mlxsw_sp_nexthop *nh) 2991 { 2992 if (!nh) { 2993 if (list_empty(&router->nexthop_list)) 2994 return NULL; 2995 else 2996 return list_first_entry(&router->nexthop_list, 2997 typeof(*nh), router_list_node); 2998 } 2999 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3000 return NULL; 3001 return list_next_entry(nh, router_list_node); 3002 } 3003 3004 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3005 { 3006 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3007 } 3008 3009 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3010 { 3011 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3012 !mlxsw_sp_nexthop_is_forward(nh)) 3013 return NULL; 3014 return nh->neigh_entry->ha; 3015 } 3016 3017 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3018 u32 *p_adj_size, u32 *p_adj_hash_index) 3019 { 3020 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3021 u32 adj_hash_index = 0; 3022 int i; 3023 3024 if (!nh->offloaded || !nhgi->adj_index_valid) 3025 return -EINVAL; 3026 3027 *p_adj_index = nhgi->adj_index; 3028 *p_adj_size = nhgi->ecmp_size; 3029 3030 for (i = 0; i < nhgi->count; i++) { 3031 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3032 3033 if (nh_iter == nh) 3034 break; 3035 if (nh_iter->offloaded) 3036 adj_hash_index += nh_iter->num_adj_entries; 3037 } 3038 3039 *p_adj_hash_index = adj_hash_index; 3040 return 0; 3041 } 3042 3043 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3044 { 3045 return nh->rif; 3046 } 3047 3048 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3049 { 3050 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3051 int i; 3052 3053 for (i = 0; i < nhgi->count; i++) { 3054 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3055 3056 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3057 return true; 3058 } 3059 return false; 3060 } 3061 3062 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3063 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3064 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3065 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3066 .automatic_shrinking = true, 3067 }; 3068 3069 static struct mlxsw_sp_nexthop_group_vr_entry * 3070 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3071 const struct mlxsw_sp_fib *fib) 3072 { 3073 struct mlxsw_sp_nexthop_group_vr_key key; 3074 3075 memset(&key, 0, sizeof(key)); 3076 key.vr_id = fib->vr->id; 3077 key.proto = fib->proto; 3078 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3079 mlxsw_sp_nexthop_group_vr_ht_params); 3080 } 3081 3082 static int 3083 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3084 const struct mlxsw_sp_fib *fib) 3085 { 3086 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3087 int err; 3088 3089 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3090 if (!vr_entry) 3091 return -ENOMEM; 3092 3093 vr_entry->key.vr_id = fib->vr->id; 3094 vr_entry->key.proto = fib->proto; 3095 refcount_set(&vr_entry->ref_count, 1); 3096 3097 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3098 mlxsw_sp_nexthop_group_vr_ht_params); 3099 if (err) 3100 goto err_hashtable_insert; 3101 3102 list_add(&vr_entry->list, &nh_grp->vr_list); 3103 3104 return 0; 3105 3106 err_hashtable_insert: 3107 kfree(vr_entry); 3108 return err; 3109 } 3110 3111 static void 3112 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3113 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3114 { 3115 list_del(&vr_entry->list); 3116 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3117 mlxsw_sp_nexthop_group_vr_ht_params); 3118 kfree(vr_entry); 3119 } 3120 3121 static int 3122 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3123 const struct mlxsw_sp_fib *fib) 3124 { 3125 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3126 3127 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3128 if (vr_entry) { 3129 refcount_inc(&vr_entry->ref_count); 3130 return 0; 3131 } 3132 3133 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3134 } 3135 3136 static void 3137 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3138 const struct mlxsw_sp_fib *fib) 3139 { 3140 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3141 3142 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3143 if (WARN_ON_ONCE(!vr_entry)) 3144 return; 3145 3146 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3147 return; 3148 3149 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3150 } 3151 3152 struct mlxsw_sp_nexthop_group_cmp_arg { 3153 enum mlxsw_sp_nexthop_group_type type; 3154 union { 3155 struct fib_info *fi; 3156 struct mlxsw_sp_fib6_entry *fib6_entry; 3157 u32 id; 3158 }; 3159 }; 3160 3161 static bool 3162 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3163 const struct in6_addr *gw, int ifindex, 3164 int weight) 3165 { 3166 int i; 3167 3168 for (i = 0; i < nh_grp->nhgi->count; i++) { 3169 const struct mlxsw_sp_nexthop *nh; 3170 3171 nh = &nh_grp->nhgi->nexthops[i]; 3172 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3173 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3174 return true; 3175 } 3176 3177 return false; 3178 } 3179 3180 static bool 3181 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3182 const struct mlxsw_sp_fib6_entry *fib6_entry) 3183 { 3184 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3185 3186 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3187 return false; 3188 3189 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3190 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3191 struct in6_addr *gw; 3192 int ifindex, weight; 3193 3194 ifindex = fib6_nh->fib_nh_dev->ifindex; 3195 weight = fib6_nh->fib_nh_weight; 3196 gw = &fib6_nh->fib_nh_gw6; 3197 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3198 weight)) 3199 return false; 3200 } 3201 3202 return true; 3203 } 3204 3205 static int 3206 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3207 { 3208 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3209 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3210 3211 if (nh_grp->type != cmp_arg->type) 3212 return 1; 3213 3214 switch (cmp_arg->type) { 3215 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3216 return cmp_arg->fi != nh_grp->ipv4.fi; 3217 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3218 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3219 cmp_arg->fib6_entry); 3220 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3221 return cmp_arg->id != nh_grp->obj.id; 3222 default: 3223 WARN_ON(1); 3224 return 1; 3225 } 3226 } 3227 3228 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3229 { 3230 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3231 const struct mlxsw_sp_nexthop *nh; 3232 struct fib_info *fi; 3233 unsigned int val; 3234 int i; 3235 3236 switch (nh_grp->type) { 3237 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3238 fi = nh_grp->ipv4.fi; 3239 return jhash(&fi, sizeof(fi), seed); 3240 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3241 val = nh_grp->nhgi->count; 3242 for (i = 0; i < nh_grp->nhgi->count; i++) { 3243 nh = &nh_grp->nhgi->nexthops[i]; 3244 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3245 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3246 } 3247 return jhash(&val, sizeof(val), seed); 3248 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3249 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3250 default: 3251 WARN_ON(1); 3252 return 0; 3253 } 3254 } 3255 3256 static u32 3257 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3258 { 3259 unsigned int val = fib6_entry->nrt6; 3260 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3261 3262 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3263 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3264 struct net_device *dev = fib6_nh->fib_nh_dev; 3265 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3266 3267 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3268 val ^= jhash(gw, sizeof(*gw), seed); 3269 } 3270 3271 return jhash(&val, sizeof(val), seed); 3272 } 3273 3274 static u32 3275 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3276 { 3277 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3278 3279 switch (cmp_arg->type) { 3280 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3281 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3282 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3283 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3284 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3285 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3286 default: 3287 WARN_ON(1); 3288 return 0; 3289 } 3290 } 3291 3292 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3293 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3294 .hashfn = mlxsw_sp_nexthop_group_hash, 3295 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3296 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3297 }; 3298 3299 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3300 struct mlxsw_sp_nexthop_group *nh_grp) 3301 { 3302 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3303 !nh_grp->nhgi->gateway) 3304 return 0; 3305 3306 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3307 &nh_grp->ht_node, 3308 mlxsw_sp_nexthop_group_ht_params); 3309 } 3310 3311 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3312 struct mlxsw_sp_nexthop_group *nh_grp) 3313 { 3314 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3315 !nh_grp->nhgi->gateway) 3316 return; 3317 3318 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3319 &nh_grp->ht_node, 3320 mlxsw_sp_nexthop_group_ht_params); 3321 } 3322 3323 static struct mlxsw_sp_nexthop_group * 3324 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3325 struct fib_info *fi) 3326 { 3327 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3328 3329 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3330 cmp_arg.fi = fi; 3331 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3332 &cmp_arg, 3333 mlxsw_sp_nexthop_group_ht_params); 3334 } 3335 3336 static struct mlxsw_sp_nexthop_group * 3337 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3338 struct mlxsw_sp_fib6_entry *fib6_entry) 3339 { 3340 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3341 3342 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3343 cmp_arg.fib6_entry = fib6_entry; 3344 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3345 &cmp_arg, 3346 mlxsw_sp_nexthop_group_ht_params); 3347 } 3348 3349 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3350 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3351 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3352 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3353 }; 3354 3355 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3356 struct mlxsw_sp_nexthop *nh) 3357 { 3358 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3359 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3360 } 3361 3362 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3363 struct mlxsw_sp_nexthop *nh) 3364 { 3365 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3366 mlxsw_sp_nexthop_ht_params); 3367 } 3368 3369 static struct mlxsw_sp_nexthop * 3370 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3371 struct mlxsw_sp_nexthop_key key) 3372 { 3373 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3374 mlxsw_sp_nexthop_ht_params); 3375 } 3376 3377 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3378 enum mlxsw_sp_l3proto proto, 3379 u16 vr_id, 3380 u32 adj_index, u16 ecmp_size, 3381 u32 new_adj_index, 3382 u16 new_ecmp_size) 3383 { 3384 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3385 3386 mlxsw_reg_raleu_pack(raleu_pl, 3387 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3388 adj_index, ecmp_size, new_adj_index, 3389 new_ecmp_size); 3390 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3391 } 3392 3393 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3394 struct mlxsw_sp_nexthop_group *nh_grp, 3395 u32 old_adj_index, u16 old_ecmp_size) 3396 { 3397 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3398 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3399 int err; 3400 3401 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3402 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3403 vr_entry->key.proto, 3404 vr_entry->key.vr_id, 3405 old_adj_index, 3406 old_ecmp_size, 3407 nhgi->adj_index, 3408 nhgi->ecmp_size); 3409 if (err) 3410 goto err_mass_update_vr; 3411 } 3412 return 0; 3413 3414 err_mass_update_vr: 3415 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3416 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3417 vr_entry->key.vr_id, 3418 nhgi->adj_index, 3419 nhgi->ecmp_size, 3420 old_adj_index, old_ecmp_size); 3421 return err; 3422 } 3423 3424 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3425 u32 adj_index, 3426 struct mlxsw_sp_nexthop *nh, 3427 bool force, char *ratr_pl) 3428 { 3429 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3430 enum mlxsw_reg_ratr_op op; 3431 u16 rif_index; 3432 3433 rif_index = nh->rif ? nh->rif->rif_index : 3434 mlxsw_sp->router->lb_rif_index; 3435 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3436 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3437 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3438 adj_index, rif_index); 3439 switch (nh->action) { 3440 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3441 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3442 break; 3443 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3444 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3445 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3446 break; 3447 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3448 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3449 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3450 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3451 break; 3452 default: 3453 WARN_ON_ONCE(1); 3454 return -EINVAL; 3455 } 3456 if (nh->counter_valid) 3457 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3458 else 3459 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3460 3461 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3462 } 3463 3464 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3465 struct mlxsw_sp_nexthop *nh, bool force, 3466 char *ratr_pl) 3467 { 3468 int i; 3469 3470 for (i = 0; i < nh->num_adj_entries; i++) { 3471 int err; 3472 3473 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3474 nh, force, ratr_pl); 3475 if (err) 3476 return err; 3477 } 3478 3479 return 0; 3480 } 3481 3482 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3483 u32 adj_index, 3484 struct mlxsw_sp_nexthop *nh, 3485 bool force, char *ratr_pl) 3486 { 3487 const struct mlxsw_sp_ipip_ops *ipip_ops; 3488 3489 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3490 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3491 force, ratr_pl); 3492 } 3493 3494 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3495 u32 adj_index, 3496 struct mlxsw_sp_nexthop *nh, bool force, 3497 char *ratr_pl) 3498 { 3499 int i; 3500 3501 for (i = 0; i < nh->num_adj_entries; i++) { 3502 int err; 3503 3504 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3505 nh, force, ratr_pl); 3506 if (err) 3507 return err; 3508 } 3509 3510 return 0; 3511 } 3512 3513 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3514 struct mlxsw_sp_nexthop *nh, bool force, 3515 char *ratr_pl) 3516 { 3517 /* When action is discard or trap, the nexthop must be 3518 * programmed as an Ethernet nexthop. 3519 */ 3520 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3521 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3522 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3523 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3524 force, ratr_pl); 3525 else 3526 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3527 force, ratr_pl); 3528 } 3529 3530 static int 3531 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3532 struct mlxsw_sp_nexthop_group_info *nhgi, 3533 bool reallocate) 3534 { 3535 char ratr_pl[MLXSW_REG_RATR_LEN]; 3536 u32 adj_index = nhgi->adj_index; /* base */ 3537 struct mlxsw_sp_nexthop *nh; 3538 int i; 3539 3540 for (i = 0; i < nhgi->count; i++) { 3541 nh = &nhgi->nexthops[i]; 3542 3543 if (!nh->should_offload) { 3544 nh->offloaded = 0; 3545 continue; 3546 } 3547 3548 if (nh->update || reallocate) { 3549 int err = 0; 3550 3551 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3552 true, ratr_pl); 3553 if (err) 3554 return err; 3555 nh->update = 0; 3556 nh->offloaded = 1; 3557 } 3558 adj_index += nh->num_adj_entries; 3559 } 3560 return 0; 3561 } 3562 3563 static int 3564 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3565 struct mlxsw_sp_nexthop_group *nh_grp) 3566 { 3567 struct mlxsw_sp_fib_entry *fib_entry; 3568 int err; 3569 3570 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3571 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3572 if (err) 3573 return err; 3574 } 3575 return 0; 3576 } 3577 3578 struct mlxsw_sp_adj_grp_size_range { 3579 u16 start; /* Inclusive */ 3580 u16 end; /* Inclusive */ 3581 }; 3582 3583 /* Ordered by range start value */ 3584 static const struct mlxsw_sp_adj_grp_size_range 3585 mlxsw_sp1_adj_grp_size_ranges[] = { 3586 { .start = 1, .end = 64 }, 3587 { .start = 512, .end = 512 }, 3588 { .start = 1024, .end = 1024 }, 3589 { .start = 2048, .end = 2048 }, 3590 { .start = 4096, .end = 4096 }, 3591 }; 3592 3593 /* Ordered by range start value */ 3594 static const struct mlxsw_sp_adj_grp_size_range 3595 mlxsw_sp2_adj_grp_size_ranges[] = { 3596 { .start = 1, .end = 128 }, 3597 { .start = 256, .end = 256 }, 3598 { .start = 512, .end = 512 }, 3599 { .start = 1024, .end = 1024 }, 3600 { .start = 2048, .end = 2048 }, 3601 { .start = 4096, .end = 4096 }, 3602 }; 3603 3604 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3605 u16 *p_adj_grp_size) 3606 { 3607 int i; 3608 3609 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3610 const struct mlxsw_sp_adj_grp_size_range *size_range; 3611 3612 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3613 3614 if (*p_adj_grp_size >= size_range->start && 3615 *p_adj_grp_size <= size_range->end) 3616 return; 3617 3618 if (*p_adj_grp_size <= size_range->end) { 3619 *p_adj_grp_size = size_range->end; 3620 return; 3621 } 3622 } 3623 } 3624 3625 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3626 u16 *p_adj_grp_size, 3627 unsigned int alloc_size) 3628 { 3629 int i; 3630 3631 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3632 const struct mlxsw_sp_adj_grp_size_range *size_range; 3633 3634 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3635 3636 if (alloc_size >= size_range->end) { 3637 *p_adj_grp_size = size_range->end; 3638 return; 3639 } 3640 } 3641 } 3642 3643 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3644 u16 *p_adj_grp_size) 3645 { 3646 unsigned int alloc_size; 3647 int err; 3648 3649 /* Round up the requested group size to the next size supported 3650 * by the device and make sure the request can be satisfied. 3651 */ 3652 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3653 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3654 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3655 *p_adj_grp_size, &alloc_size); 3656 if (err) 3657 return err; 3658 /* It is possible the allocation results in more allocated 3659 * entries than requested. Try to use as much of them as 3660 * possible. 3661 */ 3662 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3663 3664 return 0; 3665 } 3666 3667 static void 3668 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3669 { 3670 int i, g = 0, sum_norm_weight = 0; 3671 struct mlxsw_sp_nexthop *nh; 3672 3673 for (i = 0; i < nhgi->count; i++) { 3674 nh = &nhgi->nexthops[i]; 3675 3676 if (!nh->should_offload) 3677 continue; 3678 if (g > 0) 3679 g = gcd(nh->nh_weight, g); 3680 else 3681 g = nh->nh_weight; 3682 } 3683 3684 for (i = 0; i < nhgi->count; i++) { 3685 nh = &nhgi->nexthops[i]; 3686 3687 if (!nh->should_offload) 3688 continue; 3689 nh->norm_nh_weight = nh->nh_weight / g; 3690 sum_norm_weight += nh->norm_nh_weight; 3691 } 3692 3693 nhgi->sum_norm_weight = sum_norm_weight; 3694 } 3695 3696 static void 3697 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3698 { 3699 int i, weight = 0, lower_bound = 0; 3700 int total = nhgi->sum_norm_weight; 3701 u16 ecmp_size = nhgi->ecmp_size; 3702 3703 for (i = 0; i < nhgi->count; i++) { 3704 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3705 int upper_bound; 3706 3707 if (!nh->should_offload) 3708 continue; 3709 weight += nh->norm_nh_weight; 3710 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3711 nh->num_adj_entries = upper_bound - lower_bound; 3712 lower_bound = upper_bound; 3713 } 3714 } 3715 3716 static struct mlxsw_sp_nexthop * 3717 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3718 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3719 3720 static void 3721 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3722 struct mlxsw_sp_nexthop_group *nh_grp) 3723 { 3724 int i; 3725 3726 for (i = 0; i < nh_grp->nhgi->count; i++) { 3727 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3728 3729 if (nh->offloaded) 3730 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3731 else 3732 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3733 } 3734 } 3735 3736 static void 3737 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3738 struct mlxsw_sp_fib6_entry *fib6_entry) 3739 { 3740 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3741 3742 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3743 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3744 struct mlxsw_sp_nexthop *nh; 3745 3746 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3747 if (nh && nh->offloaded) 3748 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3749 else 3750 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3751 } 3752 } 3753 3754 static void 3755 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3756 struct mlxsw_sp_nexthop_group *nh_grp) 3757 { 3758 struct mlxsw_sp_fib6_entry *fib6_entry; 3759 3760 /* Unfortunately, in IPv6 the route and the nexthop are described by 3761 * the same struct, so we need to iterate over all the routes using the 3762 * nexthop group and set / clear the offload indication for them. 3763 */ 3764 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3765 common.nexthop_group_node) 3766 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3767 } 3768 3769 static void 3770 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3771 const struct mlxsw_sp_nexthop *nh, 3772 u16 bucket_index) 3773 { 3774 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3775 bool offload = false, trap = false; 3776 3777 if (nh->offloaded) { 3778 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3779 trap = true; 3780 else 3781 offload = true; 3782 } 3783 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3784 bucket_index, offload, trap); 3785 } 3786 3787 static void 3788 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3789 struct mlxsw_sp_nexthop_group *nh_grp) 3790 { 3791 int i; 3792 3793 /* Do not update the flags if the nexthop group is being destroyed 3794 * since: 3795 * 1. The nexthop objects is being deleted, in which case the flags are 3796 * irrelevant. 3797 * 2. The nexthop group was replaced by a newer group, in which case 3798 * the flags of the nexthop object were already updated based on the 3799 * new group. 3800 */ 3801 if (nh_grp->can_destroy) 3802 return; 3803 3804 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3805 nh_grp->nhgi->adj_index_valid, false); 3806 3807 /* Update flags of individual nexthop buckets in case of a resilient 3808 * nexthop group. 3809 */ 3810 if (!nh_grp->nhgi->is_resilient) 3811 return; 3812 3813 for (i = 0; i < nh_grp->nhgi->count; i++) { 3814 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3815 3816 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3817 } 3818 } 3819 3820 static void 3821 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3822 struct mlxsw_sp_nexthop_group *nh_grp) 3823 { 3824 switch (nh_grp->type) { 3825 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3826 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3827 break; 3828 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3829 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3830 break; 3831 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3832 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3833 break; 3834 } 3835 } 3836 3837 static int 3838 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3839 struct mlxsw_sp_nexthop_group *nh_grp) 3840 { 3841 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3842 u16 ecmp_size, old_ecmp_size; 3843 struct mlxsw_sp_nexthop *nh; 3844 bool offload_change = false; 3845 u32 adj_index; 3846 bool old_adj_index_valid; 3847 u32 old_adj_index; 3848 int i, err2, err; 3849 3850 if (!nhgi->gateway) 3851 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3852 3853 for (i = 0; i < nhgi->count; i++) { 3854 nh = &nhgi->nexthops[i]; 3855 3856 if (nh->should_offload != nh->offloaded) { 3857 offload_change = true; 3858 if (nh->should_offload) 3859 nh->update = 1; 3860 } 3861 } 3862 if (!offload_change) { 3863 /* Nothing was added or removed, so no need to reallocate. Just 3864 * update MAC on existing adjacency indexes. 3865 */ 3866 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3867 if (err) { 3868 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3869 goto set_trap; 3870 } 3871 /* Flags of individual nexthop buckets might need to be 3872 * updated. 3873 */ 3874 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3875 return 0; 3876 } 3877 mlxsw_sp_nexthop_group_normalize(nhgi); 3878 if (!nhgi->sum_norm_weight) { 3879 /* No neigh of this group is connected so we just set 3880 * the trap and let everthing flow through kernel. 3881 */ 3882 err = 0; 3883 goto set_trap; 3884 } 3885 3886 ecmp_size = nhgi->sum_norm_weight; 3887 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3888 if (err) 3889 /* No valid allocation size available. */ 3890 goto set_trap; 3891 3892 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3893 ecmp_size, &adj_index); 3894 if (err) { 3895 /* We ran out of KVD linear space, just set the 3896 * trap and let everything flow through kernel. 3897 */ 3898 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3899 goto set_trap; 3900 } 3901 old_adj_index_valid = nhgi->adj_index_valid; 3902 old_adj_index = nhgi->adj_index; 3903 old_ecmp_size = nhgi->ecmp_size; 3904 nhgi->adj_index_valid = 1; 3905 nhgi->adj_index = adj_index; 3906 nhgi->ecmp_size = ecmp_size; 3907 mlxsw_sp_nexthop_group_rebalance(nhgi); 3908 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3909 if (err) { 3910 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3911 goto set_trap; 3912 } 3913 3914 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3915 3916 if (!old_adj_index_valid) { 3917 /* The trap was set for fib entries, so we have to call 3918 * fib entry update to unset it and use adjacency index. 3919 */ 3920 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3921 if (err) { 3922 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3923 goto set_trap; 3924 } 3925 return 0; 3926 } 3927 3928 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3929 old_adj_index, old_ecmp_size); 3930 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3931 old_ecmp_size, old_adj_index); 3932 if (err) { 3933 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3934 goto set_trap; 3935 } 3936 3937 return 0; 3938 3939 set_trap: 3940 old_adj_index_valid = nhgi->adj_index_valid; 3941 nhgi->adj_index_valid = 0; 3942 for (i = 0; i < nhgi->count; i++) { 3943 nh = &nhgi->nexthops[i]; 3944 nh->offloaded = 0; 3945 } 3946 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3947 if (err2) 3948 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3949 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3950 if (old_adj_index_valid) 3951 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3952 nhgi->ecmp_size, nhgi->adj_index); 3953 return err; 3954 } 3955 3956 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3957 bool removing) 3958 { 3959 if (!removing) { 3960 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3961 nh->should_offload = 1; 3962 } else if (nh->nhgi->is_resilient) { 3963 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 3964 nh->should_offload = 1; 3965 } else { 3966 nh->should_offload = 0; 3967 } 3968 nh->update = 1; 3969 } 3970 3971 static int 3972 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 3973 struct mlxsw_sp_neigh_entry *neigh_entry) 3974 { 3975 struct neighbour *n, *old_n = neigh_entry->key.n; 3976 struct mlxsw_sp_nexthop *nh; 3977 bool entry_connected; 3978 u8 nud_state, dead; 3979 int err; 3980 3981 nh = list_first_entry(&neigh_entry->nexthop_list, 3982 struct mlxsw_sp_nexthop, neigh_list_node); 3983 3984 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3985 if (!n) { 3986 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3987 if (IS_ERR(n)) 3988 return PTR_ERR(n); 3989 neigh_event_send(n, NULL); 3990 } 3991 3992 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 3993 neigh_entry->key.n = n; 3994 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3995 if (err) 3996 goto err_neigh_entry_insert; 3997 3998 read_lock_bh(&n->lock); 3999 nud_state = n->nud_state; 4000 dead = n->dead; 4001 read_unlock_bh(&n->lock); 4002 entry_connected = nud_state & NUD_VALID && !dead; 4003 4004 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4005 neigh_list_node) { 4006 neigh_release(old_n); 4007 neigh_clone(n); 4008 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4009 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4010 } 4011 4012 neigh_release(n); 4013 4014 return 0; 4015 4016 err_neigh_entry_insert: 4017 neigh_entry->key.n = old_n; 4018 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4019 neigh_release(n); 4020 return err; 4021 } 4022 4023 static void 4024 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4025 struct mlxsw_sp_neigh_entry *neigh_entry, 4026 bool removing, bool dead) 4027 { 4028 struct mlxsw_sp_nexthop *nh; 4029 4030 if (list_empty(&neigh_entry->nexthop_list)) 4031 return; 4032 4033 if (dead) { 4034 int err; 4035 4036 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4037 neigh_entry); 4038 if (err) 4039 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4040 return; 4041 } 4042 4043 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4044 neigh_list_node) { 4045 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4046 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4047 } 4048 } 4049 4050 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 4051 struct mlxsw_sp_rif *rif) 4052 { 4053 if (nh->rif) 4054 return; 4055 4056 nh->rif = rif; 4057 list_add(&nh->rif_list_node, &rif->nexthop_list); 4058 } 4059 4060 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4061 { 4062 if (!nh->rif) 4063 return; 4064 4065 list_del(&nh->rif_list_node); 4066 nh->rif = NULL; 4067 } 4068 4069 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4070 struct mlxsw_sp_nexthop *nh) 4071 { 4072 struct mlxsw_sp_neigh_entry *neigh_entry; 4073 struct neighbour *n; 4074 u8 nud_state, dead; 4075 int err; 4076 4077 if (!nh->nhgi->gateway || nh->neigh_entry) 4078 return 0; 4079 4080 /* Take a reference of neigh here ensuring that neigh would 4081 * not be destructed before the nexthop entry is finished. 4082 * The reference is taken either in neigh_lookup() or 4083 * in neigh_create() in case n is not found. 4084 */ 4085 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4086 if (!n) { 4087 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4088 if (IS_ERR(n)) 4089 return PTR_ERR(n); 4090 neigh_event_send(n, NULL); 4091 } 4092 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4093 if (!neigh_entry) { 4094 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4095 if (IS_ERR(neigh_entry)) { 4096 err = -EINVAL; 4097 goto err_neigh_entry_create; 4098 } 4099 } 4100 4101 /* If that is the first nexthop connected to that neigh, add to 4102 * nexthop_neighs_list 4103 */ 4104 if (list_empty(&neigh_entry->nexthop_list)) 4105 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4106 &mlxsw_sp->router->nexthop_neighs_list); 4107 4108 nh->neigh_entry = neigh_entry; 4109 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4110 read_lock_bh(&n->lock); 4111 nud_state = n->nud_state; 4112 dead = n->dead; 4113 read_unlock_bh(&n->lock); 4114 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4115 4116 return 0; 4117 4118 err_neigh_entry_create: 4119 neigh_release(n); 4120 return err; 4121 } 4122 4123 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4124 struct mlxsw_sp_nexthop *nh) 4125 { 4126 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4127 struct neighbour *n; 4128 4129 if (!neigh_entry) 4130 return; 4131 n = neigh_entry->key.n; 4132 4133 __mlxsw_sp_nexthop_neigh_update(nh, true); 4134 list_del(&nh->neigh_list_node); 4135 nh->neigh_entry = NULL; 4136 4137 /* If that is the last nexthop connected to that neigh, remove from 4138 * nexthop_neighs_list 4139 */ 4140 if (list_empty(&neigh_entry->nexthop_list)) 4141 list_del(&neigh_entry->nexthop_neighs_list_node); 4142 4143 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4144 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4145 4146 neigh_release(n); 4147 } 4148 4149 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4150 { 4151 struct net_device *ul_dev; 4152 bool is_up; 4153 4154 rcu_read_lock(); 4155 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4156 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4157 rcu_read_unlock(); 4158 4159 return is_up; 4160 } 4161 4162 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4163 struct mlxsw_sp_nexthop *nh, 4164 struct mlxsw_sp_ipip_entry *ipip_entry) 4165 { 4166 bool removing; 4167 4168 if (!nh->nhgi->gateway || nh->ipip_entry) 4169 return; 4170 4171 nh->ipip_entry = ipip_entry; 4172 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4173 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4174 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4175 } 4176 4177 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4178 struct mlxsw_sp_nexthop *nh) 4179 { 4180 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4181 4182 if (!ipip_entry) 4183 return; 4184 4185 __mlxsw_sp_nexthop_neigh_update(nh, true); 4186 nh->ipip_entry = NULL; 4187 } 4188 4189 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4190 const struct fib_nh *fib_nh, 4191 enum mlxsw_sp_ipip_type *p_ipipt) 4192 { 4193 struct net_device *dev = fib_nh->fib_nh_dev; 4194 4195 return dev && 4196 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4197 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4198 } 4199 4200 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4201 struct mlxsw_sp_nexthop *nh, 4202 const struct net_device *dev) 4203 { 4204 const struct mlxsw_sp_ipip_ops *ipip_ops; 4205 struct mlxsw_sp_ipip_entry *ipip_entry; 4206 struct mlxsw_sp_rif *rif; 4207 int err; 4208 4209 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4210 if (ipip_entry) { 4211 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4212 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4213 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4214 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4215 return 0; 4216 } 4217 } 4218 4219 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4221 if (!rif) 4222 return 0; 4223 4224 mlxsw_sp_nexthop_rif_init(nh, rif); 4225 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4226 if (err) 4227 goto err_neigh_init; 4228 4229 return 0; 4230 4231 err_neigh_init: 4232 mlxsw_sp_nexthop_rif_fini(nh); 4233 return err; 4234 } 4235 4236 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4237 struct mlxsw_sp_nexthop *nh) 4238 { 4239 switch (nh->type) { 4240 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4241 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4242 mlxsw_sp_nexthop_rif_fini(nh); 4243 break; 4244 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4245 mlxsw_sp_nexthop_rif_fini(nh); 4246 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4247 break; 4248 } 4249 } 4250 4251 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4252 struct mlxsw_sp_nexthop_group *nh_grp, 4253 struct mlxsw_sp_nexthop *nh, 4254 struct fib_nh *fib_nh) 4255 { 4256 struct net_device *dev = fib_nh->fib_nh_dev; 4257 struct in_device *in_dev; 4258 int err; 4259 4260 nh->nhgi = nh_grp->nhgi; 4261 nh->key.fib_nh = fib_nh; 4262 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4263 nh->nh_weight = fib_nh->fib_nh_weight; 4264 #else 4265 nh->nh_weight = 1; 4266 #endif 4267 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4268 nh->neigh_tbl = &arp_tbl; 4269 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4270 if (err) 4271 return err; 4272 4273 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4274 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4275 4276 if (!dev) 4277 return 0; 4278 nh->ifindex = dev->ifindex; 4279 4280 rcu_read_lock(); 4281 in_dev = __in_dev_get_rcu(dev); 4282 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4283 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4284 rcu_read_unlock(); 4285 return 0; 4286 } 4287 rcu_read_unlock(); 4288 4289 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4290 if (err) 4291 goto err_nexthop_neigh_init; 4292 4293 return 0; 4294 4295 err_nexthop_neigh_init: 4296 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4297 return err; 4298 } 4299 4300 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4301 struct mlxsw_sp_nexthop *nh) 4302 { 4303 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4304 list_del(&nh->router_list_node); 4305 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4306 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4307 } 4308 4309 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4310 unsigned long event, struct fib_nh *fib_nh) 4311 { 4312 struct mlxsw_sp_nexthop_key key; 4313 struct mlxsw_sp_nexthop *nh; 4314 4315 key.fib_nh = fib_nh; 4316 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4317 if (!nh) 4318 return; 4319 4320 switch (event) { 4321 case FIB_EVENT_NH_ADD: 4322 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4323 break; 4324 case FIB_EVENT_NH_DEL: 4325 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4326 break; 4327 } 4328 4329 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4330 } 4331 4332 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4333 struct mlxsw_sp_rif *rif) 4334 { 4335 struct mlxsw_sp_nexthop *nh; 4336 bool removing; 4337 4338 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4339 switch (nh->type) { 4340 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4341 removing = false; 4342 break; 4343 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4344 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4345 break; 4346 default: 4347 WARN_ON(1); 4348 continue; 4349 } 4350 4351 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4352 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4353 } 4354 } 4355 4356 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4357 struct mlxsw_sp_rif *old_rif, 4358 struct mlxsw_sp_rif *new_rif) 4359 { 4360 struct mlxsw_sp_nexthop *nh; 4361 4362 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4363 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4364 nh->rif = new_rif; 4365 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4366 } 4367 4368 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4369 struct mlxsw_sp_rif *rif) 4370 { 4371 struct mlxsw_sp_nexthop *nh, *tmp; 4372 4373 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4374 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4375 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4376 } 4377 } 4378 4379 static void 4380 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4381 const struct mlxsw_sp_nexthop_group *nh_grp, 4382 unsigned long *activity) 4383 { 4384 char *ratrad_pl; 4385 int i, err; 4386 4387 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4388 if (!ratrad_pl) 4389 return; 4390 4391 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4392 nh_grp->nhgi->count); 4393 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4394 if (err) 4395 goto out; 4396 4397 for (i = 0; i < nh_grp->nhgi->count; i++) { 4398 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4399 continue; 4400 bitmap_set(activity, i, 1); 4401 } 4402 4403 out: 4404 kfree(ratrad_pl); 4405 } 4406 4407 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4408 4409 static void 4410 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4411 const struct mlxsw_sp_nexthop_group *nh_grp) 4412 { 4413 unsigned long *activity; 4414 4415 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4416 if (!activity) 4417 return; 4418 4419 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4420 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4421 nh_grp->nhgi->count, activity); 4422 4423 bitmap_free(activity); 4424 } 4425 4426 static void 4427 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4428 { 4429 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4430 4431 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4432 msecs_to_jiffies(interval)); 4433 } 4434 4435 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4436 { 4437 struct mlxsw_sp_nexthop_group_info *nhgi; 4438 struct mlxsw_sp_router *router; 4439 bool reschedule = false; 4440 4441 router = container_of(work, struct mlxsw_sp_router, 4442 nh_grp_activity_dw.work); 4443 4444 mutex_lock(&router->lock); 4445 4446 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4447 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4448 reschedule = true; 4449 } 4450 4451 mutex_unlock(&router->lock); 4452 4453 if (!reschedule) 4454 return; 4455 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4456 } 4457 4458 static int 4459 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4460 const struct nh_notifier_single_info *nh, 4461 struct netlink_ext_ack *extack) 4462 { 4463 int err = -EINVAL; 4464 4465 if (nh->is_fdb) 4466 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4467 else if (nh->has_encap) 4468 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4469 else 4470 err = 0; 4471 4472 return err; 4473 } 4474 4475 static int 4476 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4477 const struct nh_notifier_single_info *nh, 4478 struct netlink_ext_ack *extack) 4479 { 4480 int err; 4481 4482 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4483 if (err) 4484 return err; 4485 4486 /* Device only nexthops with an IPIP device are programmed as 4487 * encapsulating adjacency entries. 4488 */ 4489 if (!nh->gw_family && !nh->is_reject && 4490 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4491 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4492 return -EINVAL; 4493 } 4494 4495 return 0; 4496 } 4497 4498 static int 4499 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4500 const struct nh_notifier_grp_info *nh_grp, 4501 struct netlink_ext_ack *extack) 4502 { 4503 int i; 4504 4505 if (nh_grp->is_fdb) { 4506 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4507 return -EINVAL; 4508 } 4509 4510 for (i = 0; i < nh_grp->num_nh; i++) { 4511 const struct nh_notifier_single_info *nh; 4512 int err; 4513 4514 nh = &nh_grp->nh_entries[i].nh; 4515 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4516 extack); 4517 if (err) 4518 return err; 4519 } 4520 4521 return 0; 4522 } 4523 4524 static int 4525 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4526 const struct nh_notifier_res_table_info *nh_res_table, 4527 struct netlink_ext_ack *extack) 4528 { 4529 unsigned int alloc_size; 4530 bool valid_size = false; 4531 int err, i; 4532 4533 if (nh_res_table->num_nh_buckets < 32) { 4534 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4535 return -EINVAL; 4536 } 4537 4538 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4539 const struct mlxsw_sp_adj_grp_size_range *size_range; 4540 4541 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4542 4543 if (nh_res_table->num_nh_buckets >= size_range->start && 4544 nh_res_table->num_nh_buckets <= size_range->end) { 4545 valid_size = true; 4546 break; 4547 } 4548 } 4549 4550 if (!valid_size) { 4551 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4552 return -EINVAL; 4553 } 4554 4555 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4556 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4557 nh_res_table->num_nh_buckets, 4558 &alloc_size); 4559 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4560 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4561 return -EINVAL; 4562 } 4563 4564 return 0; 4565 } 4566 4567 static int 4568 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4569 const struct nh_notifier_res_table_info *nh_res_table, 4570 struct netlink_ext_ack *extack) 4571 { 4572 int err; 4573 u16 i; 4574 4575 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4576 nh_res_table, 4577 extack); 4578 if (err) 4579 return err; 4580 4581 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4582 const struct nh_notifier_single_info *nh; 4583 int err; 4584 4585 nh = &nh_res_table->nhs[i]; 4586 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4587 extack); 4588 if (err) 4589 return err; 4590 } 4591 4592 return 0; 4593 } 4594 4595 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4596 unsigned long event, 4597 struct nh_notifier_info *info) 4598 { 4599 struct nh_notifier_single_info *nh; 4600 4601 if (event != NEXTHOP_EVENT_REPLACE && 4602 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4603 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4604 return 0; 4605 4606 switch (info->type) { 4607 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4608 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4609 info->extack); 4610 case NH_NOTIFIER_INFO_TYPE_GRP: 4611 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4612 info->nh_grp, 4613 info->extack); 4614 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4615 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4616 info->nh_res_table, 4617 info->extack); 4618 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4619 nh = &info->nh_res_bucket->new_nh; 4620 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4621 info->extack); 4622 default: 4623 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4624 return -EOPNOTSUPP; 4625 } 4626 } 4627 4628 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4629 const struct nh_notifier_info *info) 4630 { 4631 const struct net_device *dev; 4632 4633 switch (info->type) { 4634 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4635 dev = info->nh->dev; 4636 return info->nh->gw_family || info->nh->is_reject || 4637 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4638 case NH_NOTIFIER_INFO_TYPE_GRP: 4639 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4640 /* Already validated earlier. */ 4641 return true; 4642 default: 4643 return false; 4644 } 4645 } 4646 4647 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4648 struct mlxsw_sp_nexthop *nh) 4649 { 4650 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4651 4652 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4653 nh->should_offload = 1; 4654 /* While nexthops that discard packets do not forward packets 4655 * via an egress RIF, they still need to be programmed using a 4656 * valid RIF, so use the loopback RIF created during init. 4657 */ 4658 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4659 } 4660 4661 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4662 struct mlxsw_sp_nexthop *nh) 4663 { 4664 nh->rif = NULL; 4665 nh->should_offload = 0; 4666 } 4667 4668 static int 4669 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4670 struct mlxsw_sp_nexthop_group *nh_grp, 4671 struct mlxsw_sp_nexthop *nh, 4672 struct nh_notifier_single_info *nh_obj, int weight) 4673 { 4674 struct net_device *dev = nh_obj->dev; 4675 int err; 4676 4677 nh->nhgi = nh_grp->nhgi; 4678 nh->nh_weight = weight; 4679 4680 switch (nh_obj->gw_family) { 4681 case AF_INET: 4682 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4683 nh->neigh_tbl = &arp_tbl; 4684 break; 4685 case AF_INET6: 4686 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4687 #if IS_ENABLED(CONFIG_IPV6) 4688 nh->neigh_tbl = &nd_tbl; 4689 #endif 4690 break; 4691 } 4692 4693 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4694 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4695 nh->ifindex = dev->ifindex; 4696 4697 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4698 if (err) 4699 goto err_type_init; 4700 4701 if (nh_obj->is_reject) 4702 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4703 4704 /* In a resilient nexthop group, all the nexthops must be written to 4705 * the adjacency table. Even if they do not have a valid neighbour or 4706 * RIF. 4707 */ 4708 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4709 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4710 nh->should_offload = 1; 4711 } 4712 4713 return 0; 4714 4715 err_type_init: 4716 list_del(&nh->router_list_node); 4717 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4718 return err; 4719 } 4720 4721 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4722 struct mlxsw_sp_nexthop *nh) 4723 { 4724 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4725 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4726 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4727 list_del(&nh->router_list_node); 4728 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4729 nh->should_offload = 0; 4730 } 4731 4732 static int 4733 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4734 struct mlxsw_sp_nexthop_group *nh_grp, 4735 struct nh_notifier_info *info) 4736 { 4737 struct mlxsw_sp_nexthop_group_info *nhgi; 4738 struct mlxsw_sp_nexthop *nh; 4739 bool is_resilient = false; 4740 unsigned int nhs; 4741 int err, i; 4742 4743 switch (info->type) { 4744 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4745 nhs = 1; 4746 break; 4747 case NH_NOTIFIER_INFO_TYPE_GRP: 4748 nhs = info->nh_grp->num_nh; 4749 break; 4750 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4751 nhs = info->nh_res_table->num_nh_buckets; 4752 is_resilient = true; 4753 break; 4754 default: 4755 return -EINVAL; 4756 } 4757 4758 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4759 if (!nhgi) 4760 return -ENOMEM; 4761 nh_grp->nhgi = nhgi; 4762 nhgi->nh_grp = nh_grp; 4763 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4764 nhgi->is_resilient = is_resilient; 4765 nhgi->count = nhs; 4766 for (i = 0; i < nhgi->count; i++) { 4767 struct nh_notifier_single_info *nh_obj; 4768 int weight; 4769 4770 nh = &nhgi->nexthops[i]; 4771 switch (info->type) { 4772 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4773 nh_obj = info->nh; 4774 weight = 1; 4775 break; 4776 case NH_NOTIFIER_INFO_TYPE_GRP: 4777 nh_obj = &info->nh_grp->nh_entries[i].nh; 4778 weight = info->nh_grp->nh_entries[i].weight; 4779 break; 4780 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4781 nh_obj = &info->nh_res_table->nhs[i]; 4782 weight = 1; 4783 break; 4784 default: 4785 err = -EINVAL; 4786 goto err_nexthop_obj_init; 4787 } 4788 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4789 weight); 4790 if (err) 4791 goto err_nexthop_obj_init; 4792 } 4793 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4794 if (err) { 4795 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4796 goto err_group_refresh; 4797 } 4798 4799 /* Add resilient nexthop groups to a list so that the activity of their 4800 * nexthop buckets will be periodically queried and cleared. 4801 */ 4802 if (nhgi->is_resilient) { 4803 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4804 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 4805 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 4806 } 4807 4808 return 0; 4809 4810 err_group_refresh: 4811 i = nhgi->count; 4812 err_nexthop_obj_init: 4813 for (i--; i >= 0; i--) { 4814 nh = &nhgi->nexthops[i]; 4815 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4816 } 4817 kfree(nhgi); 4818 return err; 4819 } 4820 4821 static void 4822 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4823 struct mlxsw_sp_nexthop_group *nh_grp) 4824 { 4825 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4826 struct mlxsw_sp_router *router = mlxsw_sp->router; 4827 int i; 4828 4829 if (nhgi->is_resilient) { 4830 list_del(&nhgi->list); 4831 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4832 cancel_delayed_work(&router->nh_grp_activity_dw); 4833 } 4834 4835 for (i = nhgi->count - 1; i >= 0; i--) { 4836 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4837 4838 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4839 } 4840 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4841 WARN_ON_ONCE(nhgi->adj_index_valid); 4842 kfree(nhgi); 4843 } 4844 4845 static struct mlxsw_sp_nexthop_group * 4846 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4847 struct nh_notifier_info *info) 4848 { 4849 struct mlxsw_sp_nexthop_group *nh_grp; 4850 int err; 4851 4852 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4853 if (!nh_grp) 4854 return ERR_PTR(-ENOMEM); 4855 INIT_LIST_HEAD(&nh_grp->vr_list); 4856 err = rhashtable_init(&nh_grp->vr_ht, 4857 &mlxsw_sp_nexthop_group_vr_ht_params); 4858 if (err) 4859 goto err_nexthop_group_vr_ht_init; 4860 INIT_LIST_HEAD(&nh_grp->fib_list); 4861 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4862 nh_grp->obj.id = info->id; 4863 4864 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4865 if (err) 4866 goto err_nexthop_group_info_init; 4867 4868 nh_grp->can_destroy = false; 4869 4870 return nh_grp; 4871 4872 err_nexthop_group_info_init: 4873 rhashtable_destroy(&nh_grp->vr_ht); 4874 err_nexthop_group_vr_ht_init: 4875 kfree(nh_grp); 4876 return ERR_PTR(err); 4877 } 4878 4879 static void 4880 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4881 struct mlxsw_sp_nexthop_group *nh_grp) 4882 { 4883 if (!nh_grp->can_destroy) 4884 return; 4885 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4886 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4887 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4888 rhashtable_destroy(&nh_grp->vr_ht); 4889 kfree(nh_grp); 4890 } 4891 4892 static struct mlxsw_sp_nexthop_group * 4893 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4894 { 4895 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4896 4897 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4898 cmp_arg.id = id; 4899 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4900 &cmp_arg, 4901 mlxsw_sp_nexthop_group_ht_params); 4902 } 4903 4904 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 4905 struct mlxsw_sp_nexthop_group *nh_grp) 4906 { 4907 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4908 } 4909 4910 static int 4911 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 4912 struct mlxsw_sp_nexthop_group *nh_grp, 4913 struct mlxsw_sp_nexthop_group *old_nh_grp, 4914 struct netlink_ext_ack *extack) 4915 { 4916 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 4917 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 4918 int err; 4919 4920 old_nh_grp->nhgi = new_nhgi; 4921 new_nhgi->nh_grp = old_nh_grp; 4922 nh_grp->nhgi = old_nhgi; 4923 old_nhgi->nh_grp = nh_grp; 4924 4925 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4926 /* Both the old adjacency index and the new one are valid. 4927 * Routes are currently using the old one. Tell the device to 4928 * replace the old adjacency index with the new one. 4929 */ 4930 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 4931 old_nhgi->adj_index, 4932 old_nhgi->ecmp_size); 4933 if (err) { 4934 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 4935 goto err_out; 4936 } 4937 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 4938 /* The old adjacency index is valid, while the new one is not. 4939 * Iterate over all the routes using the group and change them 4940 * to trap packets to the CPU. 4941 */ 4942 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4943 if (err) { 4944 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 4945 goto err_out; 4946 } 4947 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4948 /* The old adjacency index is invalid, while the new one is. 4949 * Iterate over all the routes using the group and change them 4950 * to forward packets using the new valid index. 4951 */ 4952 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4953 if (err) { 4954 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 4955 goto err_out; 4956 } 4957 } 4958 4959 /* Make sure the flags are set / cleared based on the new nexthop group 4960 * information. 4961 */ 4962 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 4963 4964 /* At this point 'nh_grp' is just a shell that is not used by anyone 4965 * and its nexthop group info is the old info that was just replaced 4966 * with the new one. Remove it. 4967 */ 4968 nh_grp->can_destroy = true; 4969 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4970 4971 return 0; 4972 4973 err_out: 4974 old_nhgi->nh_grp = old_nh_grp; 4975 nh_grp->nhgi = new_nhgi; 4976 new_nhgi->nh_grp = nh_grp; 4977 old_nh_grp->nhgi = old_nhgi; 4978 return err; 4979 } 4980 4981 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 4982 struct nh_notifier_info *info) 4983 { 4984 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 4985 struct netlink_ext_ack *extack = info->extack; 4986 int err; 4987 4988 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 4989 if (IS_ERR(nh_grp)) 4990 return PTR_ERR(nh_grp); 4991 4992 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4993 if (!old_nh_grp) 4994 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 4995 else 4996 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 4997 old_nh_grp, extack); 4998 4999 if (err) { 5000 nh_grp->can_destroy = true; 5001 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5002 } 5003 5004 return err; 5005 } 5006 5007 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5008 struct nh_notifier_info *info) 5009 { 5010 struct mlxsw_sp_nexthop_group *nh_grp; 5011 5012 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5013 if (!nh_grp) 5014 return; 5015 5016 nh_grp->can_destroy = true; 5017 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5018 5019 /* If the group still has routes using it, then defer the delete 5020 * operation until the last route using it is deleted. 5021 */ 5022 if (!list_empty(&nh_grp->fib_list)) 5023 return; 5024 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5025 } 5026 5027 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5028 u32 adj_index, char *ratr_pl) 5029 { 5030 MLXSW_REG_ZERO(ratr, ratr_pl); 5031 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5032 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5033 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5034 5035 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5036 } 5037 5038 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5039 { 5040 /* Clear the opcode and activity on both the old and new payload as 5041 * they are irrelevant for the comparison. 5042 */ 5043 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5044 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5045 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5046 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5047 5048 /* If the contents of the adjacency entry are consistent with the 5049 * replacement request, then replacement was successful. 5050 */ 5051 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5052 return 0; 5053 5054 return -EINVAL; 5055 } 5056 5057 static int 5058 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5059 struct mlxsw_sp_nexthop *nh, 5060 struct nh_notifier_info *info) 5061 { 5062 u16 bucket_index = info->nh_res_bucket->bucket_index; 5063 struct netlink_ext_ack *extack = info->extack; 5064 bool force = info->nh_res_bucket->force; 5065 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5066 char ratr_pl[MLXSW_REG_RATR_LEN]; 5067 u32 adj_index; 5068 int err; 5069 5070 /* No point in trying an atomic replacement if the idle timer interval 5071 * is smaller than the interval in which we query and clear activity. 5072 */ 5073 if (!force && info->nh_res_bucket->idle_timer_ms < 5074 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5075 force = true; 5076 5077 adj_index = nh->nhgi->adj_index + bucket_index; 5078 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5079 if (err) { 5080 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5081 return err; 5082 } 5083 5084 if (!force) { 5085 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5086 ratr_pl_new); 5087 if (err) { 5088 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5089 return err; 5090 } 5091 5092 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5093 if (err) { 5094 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5095 return err; 5096 } 5097 } 5098 5099 nh->update = 0; 5100 nh->offloaded = 1; 5101 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5102 5103 return 0; 5104 } 5105 5106 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5107 struct nh_notifier_info *info) 5108 { 5109 u16 bucket_index = info->nh_res_bucket->bucket_index; 5110 struct netlink_ext_ack *extack = info->extack; 5111 struct mlxsw_sp_nexthop_group_info *nhgi; 5112 struct nh_notifier_single_info *nh_obj; 5113 struct mlxsw_sp_nexthop_group *nh_grp; 5114 struct mlxsw_sp_nexthop *nh; 5115 int err; 5116 5117 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5118 if (!nh_grp) { 5119 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5120 return -EINVAL; 5121 } 5122 5123 nhgi = nh_grp->nhgi; 5124 5125 if (bucket_index >= nhgi->count) { 5126 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5127 return -EINVAL; 5128 } 5129 5130 nh = &nhgi->nexthops[bucket_index]; 5131 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5132 5133 nh_obj = &info->nh_res_bucket->new_nh; 5134 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5135 if (err) { 5136 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5137 goto err_nexthop_obj_init; 5138 } 5139 5140 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5141 if (err) 5142 goto err_nexthop_obj_bucket_adj_update; 5143 5144 return 0; 5145 5146 err_nexthop_obj_bucket_adj_update: 5147 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5148 err_nexthop_obj_init: 5149 nh_obj = &info->nh_res_bucket->old_nh; 5150 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5151 /* The old adjacency entry was not overwritten */ 5152 nh->update = 0; 5153 nh->offloaded = 1; 5154 return err; 5155 } 5156 5157 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5158 unsigned long event, void *ptr) 5159 { 5160 struct nh_notifier_info *info = ptr; 5161 struct mlxsw_sp_router *router; 5162 int err = 0; 5163 5164 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5165 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5166 if (err) 5167 goto out; 5168 5169 mutex_lock(&router->lock); 5170 5171 switch (event) { 5172 case NEXTHOP_EVENT_REPLACE: 5173 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5174 break; 5175 case NEXTHOP_EVENT_DEL: 5176 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5177 break; 5178 case NEXTHOP_EVENT_BUCKET_REPLACE: 5179 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5180 info); 5181 break; 5182 default: 5183 break; 5184 } 5185 5186 mutex_unlock(&router->lock); 5187 5188 out: 5189 return notifier_from_errno(err); 5190 } 5191 5192 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5193 struct fib_info *fi) 5194 { 5195 const struct fib_nh *nh = fib_info_nh(fi, 0); 5196 5197 return nh->fib_nh_scope == RT_SCOPE_LINK || 5198 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5199 } 5200 5201 static int 5202 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5203 struct mlxsw_sp_nexthop_group *nh_grp) 5204 { 5205 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5206 struct mlxsw_sp_nexthop_group_info *nhgi; 5207 struct mlxsw_sp_nexthop *nh; 5208 int err, i; 5209 5210 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5211 if (!nhgi) 5212 return -ENOMEM; 5213 nh_grp->nhgi = nhgi; 5214 nhgi->nh_grp = nh_grp; 5215 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5216 nhgi->count = nhs; 5217 for (i = 0; i < nhgi->count; i++) { 5218 struct fib_nh *fib_nh; 5219 5220 nh = &nhgi->nexthops[i]; 5221 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5222 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5223 if (err) 5224 goto err_nexthop4_init; 5225 } 5226 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5227 if (err) 5228 goto err_group_refresh; 5229 5230 return 0; 5231 5232 err_group_refresh: 5233 i = nhgi->count; 5234 err_nexthop4_init: 5235 for (i--; i >= 0; i--) { 5236 nh = &nhgi->nexthops[i]; 5237 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5238 } 5239 kfree(nhgi); 5240 return err; 5241 } 5242 5243 static void 5244 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5245 struct mlxsw_sp_nexthop_group *nh_grp) 5246 { 5247 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5248 int i; 5249 5250 for (i = nhgi->count - 1; i >= 0; i--) { 5251 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5252 5253 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5254 } 5255 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5256 WARN_ON_ONCE(nhgi->adj_index_valid); 5257 kfree(nhgi); 5258 } 5259 5260 static struct mlxsw_sp_nexthop_group * 5261 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5262 { 5263 struct mlxsw_sp_nexthop_group *nh_grp; 5264 int err; 5265 5266 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5267 if (!nh_grp) 5268 return ERR_PTR(-ENOMEM); 5269 INIT_LIST_HEAD(&nh_grp->vr_list); 5270 err = rhashtable_init(&nh_grp->vr_ht, 5271 &mlxsw_sp_nexthop_group_vr_ht_params); 5272 if (err) 5273 goto err_nexthop_group_vr_ht_init; 5274 INIT_LIST_HEAD(&nh_grp->fib_list); 5275 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5276 nh_grp->ipv4.fi = fi; 5277 fib_info_hold(fi); 5278 5279 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5280 if (err) 5281 goto err_nexthop_group_info_init; 5282 5283 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5284 if (err) 5285 goto err_nexthop_group_insert; 5286 5287 nh_grp->can_destroy = true; 5288 5289 return nh_grp; 5290 5291 err_nexthop_group_insert: 5292 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5293 err_nexthop_group_info_init: 5294 fib_info_put(fi); 5295 rhashtable_destroy(&nh_grp->vr_ht); 5296 err_nexthop_group_vr_ht_init: 5297 kfree(nh_grp); 5298 return ERR_PTR(err); 5299 } 5300 5301 static void 5302 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5303 struct mlxsw_sp_nexthop_group *nh_grp) 5304 { 5305 if (!nh_grp->can_destroy) 5306 return; 5307 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5308 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5309 fib_info_put(nh_grp->ipv4.fi); 5310 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5311 rhashtable_destroy(&nh_grp->vr_ht); 5312 kfree(nh_grp); 5313 } 5314 5315 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5316 struct mlxsw_sp_fib_entry *fib_entry, 5317 struct fib_info *fi) 5318 { 5319 struct mlxsw_sp_nexthop_group *nh_grp; 5320 5321 if (fi->nh) { 5322 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5323 fi->nh->id); 5324 if (WARN_ON_ONCE(!nh_grp)) 5325 return -EINVAL; 5326 goto out; 5327 } 5328 5329 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5330 if (!nh_grp) { 5331 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5332 if (IS_ERR(nh_grp)) 5333 return PTR_ERR(nh_grp); 5334 } 5335 out: 5336 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5337 fib_entry->nh_group = nh_grp; 5338 return 0; 5339 } 5340 5341 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5342 struct mlxsw_sp_fib_entry *fib_entry) 5343 { 5344 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5345 5346 list_del(&fib_entry->nexthop_group_node); 5347 if (!list_empty(&nh_grp->fib_list)) 5348 return; 5349 5350 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5351 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5352 return; 5353 } 5354 5355 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5356 } 5357 5358 static bool 5359 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5360 { 5361 struct mlxsw_sp_fib4_entry *fib4_entry; 5362 5363 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5364 common); 5365 return !fib4_entry->tos; 5366 } 5367 5368 static bool 5369 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5370 { 5371 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5372 5373 switch (fib_entry->fib_node->fib->proto) { 5374 case MLXSW_SP_L3_PROTO_IPV4: 5375 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5376 return false; 5377 break; 5378 case MLXSW_SP_L3_PROTO_IPV6: 5379 break; 5380 } 5381 5382 switch (fib_entry->type) { 5383 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5384 return !!nh_group->nhgi->adj_index_valid; 5385 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5386 return !!nh_group->nhgi->nh_rif; 5387 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5388 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5389 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5390 return true; 5391 default: 5392 return false; 5393 } 5394 } 5395 5396 static struct mlxsw_sp_nexthop * 5397 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5398 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5399 { 5400 int i; 5401 5402 for (i = 0; i < nh_grp->nhgi->count; i++) { 5403 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5404 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5405 5406 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5407 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5408 &rt->fib6_nh->fib_nh_gw6)) 5409 return nh; 5410 continue; 5411 } 5412 5413 return NULL; 5414 } 5415 5416 static void 5417 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5418 struct fib_entry_notifier_info *fen_info) 5419 { 5420 u32 *p_dst = (u32 *) &fen_info->dst; 5421 struct fib_rt_info fri; 5422 5423 fri.fi = fen_info->fi; 5424 fri.tb_id = fen_info->tb_id; 5425 fri.dst = cpu_to_be32(*p_dst); 5426 fri.dst_len = fen_info->dst_len; 5427 fri.tos = fen_info->tos; 5428 fri.type = fen_info->type; 5429 fri.offload = false; 5430 fri.trap = false; 5431 fri.offload_failed = true; 5432 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5433 } 5434 5435 static void 5436 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5437 struct mlxsw_sp_fib_entry *fib_entry) 5438 { 5439 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5440 int dst_len = fib_entry->fib_node->key.prefix_len; 5441 struct mlxsw_sp_fib4_entry *fib4_entry; 5442 struct fib_rt_info fri; 5443 bool should_offload; 5444 5445 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5446 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5447 common); 5448 fri.fi = fib4_entry->fi; 5449 fri.tb_id = fib4_entry->tb_id; 5450 fri.dst = cpu_to_be32(*p_dst); 5451 fri.dst_len = dst_len; 5452 fri.tos = fib4_entry->tos; 5453 fri.type = fib4_entry->type; 5454 fri.offload = should_offload; 5455 fri.trap = !should_offload; 5456 fri.offload_failed = false; 5457 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5458 } 5459 5460 static void 5461 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5462 struct mlxsw_sp_fib_entry *fib_entry) 5463 { 5464 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5465 int dst_len = fib_entry->fib_node->key.prefix_len; 5466 struct mlxsw_sp_fib4_entry *fib4_entry; 5467 struct fib_rt_info fri; 5468 5469 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5470 common); 5471 fri.fi = fib4_entry->fi; 5472 fri.tb_id = fib4_entry->tb_id; 5473 fri.dst = cpu_to_be32(*p_dst); 5474 fri.dst_len = dst_len; 5475 fri.tos = fib4_entry->tos; 5476 fri.type = fib4_entry->type; 5477 fri.offload = false; 5478 fri.trap = false; 5479 fri.offload_failed = false; 5480 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5481 } 5482 5483 #if IS_ENABLED(CONFIG_IPV6) 5484 static void 5485 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5486 struct fib6_info **rt_arr, 5487 unsigned int nrt6) 5488 { 5489 int i; 5490 5491 /* In IPv6 a multipath route is represented using multiple routes, so 5492 * we need to set the flags on all of them. 5493 */ 5494 for (i = 0; i < nrt6; i++) 5495 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5496 false, false, true); 5497 } 5498 #else 5499 static void 5500 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5501 struct fib6_info **rt_arr, 5502 unsigned int nrt6) 5503 { 5504 } 5505 #endif 5506 5507 #if IS_ENABLED(CONFIG_IPV6) 5508 static void 5509 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5510 struct mlxsw_sp_fib_entry *fib_entry) 5511 { 5512 struct mlxsw_sp_fib6_entry *fib6_entry; 5513 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5514 bool should_offload; 5515 5516 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5517 5518 /* In IPv6 a multipath route is represented using multiple routes, so 5519 * we need to set the flags on all of them. 5520 */ 5521 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5522 common); 5523 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5524 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5525 should_offload, !should_offload, false); 5526 } 5527 #else 5528 static void 5529 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5530 struct mlxsw_sp_fib_entry *fib_entry) 5531 { 5532 } 5533 #endif 5534 5535 #if IS_ENABLED(CONFIG_IPV6) 5536 static void 5537 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5538 struct mlxsw_sp_fib_entry *fib_entry) 5539 { 5540 struct mlxsw_sp_fib6_entry *fib6_entry; 5541 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5542 5543 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5544 common); 5545 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5546 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5547 false, false, false); 5548 } 5549 #else 5550 static void 5551 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5552 struct mlxsw_sp_fib_entry *fib_entry) 5553 { 5554 } 5555 #endif 5556 5557 static void 5558 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5559 struct mlxsw_sp_fib_entry *fib_entry) 5560 { 5561 switch (fib_entry->fib_node->fib->proto) { 5562 case MLXSW_SP_L3_PROTO_IPV4: 5563 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5564 break; 5565 case MLXSW_SP_L3_PROTO_IPV6: 5566 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5567 break; 5568 } 5569 } 5570 5571 static void 5572 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5573 struct mlxsw_sp_fib_entry *fib_entry) 5574 { 5575 switch (fib_entry->fib_node->fib->proto) { 5576 case MLXSW_SP_L3_PROTO_IPV4: 5577 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5578 break; 5579 case MLXSW_SP_L3_PROTO_IPV6: 5580 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5581 break; 5582 } 5583 } 5584 5585 static void 5586 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5587 struct mlxsw_sp_fib_entry *fib_entry, 5588 enum mlxsw_sp_fib_entry_op op) 5589 { 5590 switch (op) { 5591 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5592 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5593 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5594 break; 5595 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5596 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5597 break; 5598 default: 5599 break; 5600 } 5601 } 5602 5603 struct mlxsw_sp_fib_entry_op_ctx_basic { 5604 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5605 }; 5606 5607 static void 5608 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5609 enum mlxsw_sp_l3proto proto, 5610 enum mlxsw_sp_fib_entry_op op, 5611 u16 virtual_router, u8 prefix_len, 5612 unsigned char *addr, 5613 struct mlxsw_sp_fib_entry_priv *priv) 5614 { 5615 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5616 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5617 char *ralue_pl = op_ctx_basic->ralue_pl; 5618 enum mlxsw_reg_ralue_op ralue_op; 5619 5620 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5621 5622 switch (op) { 5623 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5624 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5625 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5626 break; 5627 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5628 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5629 break; 5630 default: 5631 WARN_ON_ONCE(1); 5632 return; 5633 } 5634 5635 switch (proto) { 5636 case MLXSW_SP_L3_PROTO_IPV4: 5637 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5638 virtual_router, prefix_len, (u32 *) addr); 5639 break; 5640 case MLXSW_SP_L3_PROTO_IPV6: 5641 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5642 virtual_router, prefix_len, addr); 5643 break; 5644 } 5645 } 5646 5647 static void 5648 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5649 enum mlxsw_reg_ralue_trap_action trap_action, 5650 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5651 { 5652 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5653 5654 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5655 trap_id, adjacency_index, ecmp_size); 5656 } 5657 5658 static void 5659 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5660 enum mlxsw_reg_ralue_trap_action trap_action, 5661 u16 trap_id, u16 local_erif) 5662 { 5663 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5664 5665 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5666 trap_id, local_erif); 5667 } 5668 5669 static void 5670 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5671 { 5672 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5673 5674 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5675 } 5676 5677 static void 5678 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5679 u32 tunnel_ptr) 5680 { 5681 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5682 5683 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5684 } 5685 5686 static int 5687 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5688 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5689 bool *postponed_for_bulk) 5690 { 5691 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5692 5693 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5694 op_ctx_basic->ralue_pl); 5695 } 5696 5697 static bool 5698 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5699 { 5700 return true; 5701 } 5702 5703 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5704 struct mlxsw_sp_fib_entry *fib_entry, 5705 enum mlxsw_sp_fib_entry_op op) 5706 { 5707 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5708 5709 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5710 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5711 fib_entry->fib_node->key.prefix_len, 5712 fib_entry->fib_node->key.addr, 5713 fib_entry->priv); 5714 } 5715 5716 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5717 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5718 const struct mlxsw_sp_router_ll_ops *ll_ops) 5719 { 5720 bool postponed_for_bulk = false; 5721 int err; 5722 5723 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5724 if (!postponed_for_bulk) 5725 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5726 return err; 5727 } 5728 5729 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp) 5730 { 5731 enum mlxsw_reg_ratr_trap_action trap_action; 5732 char ratr_pl[MLXSW_REG_RATR_LEN]; 5733 int err; 5734 5735 if (mlxsw_sp->router->adj_discard_index_valid) 5736 return 0; 5737 5738 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5739 &mlxsw_sp->router->adj_discard_index); 5740 if (err) 5741 return err; 5742 5743 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 5744 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 5745 MLXSW_REG_RATR_TYPE_ETHERNET, 5746 mlxsw_sp->router->adj_discard_index, 5747 mlxsw_sp->router->lb_rif_index); 5748 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 5749 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 5750 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5751 if (err) 5752 goto err_ratr_write; 5753 5754 mlxsw_sp->router->adj_discard_index_valid = true; 5755 5756 return 0; 5757 5758 err_ratr_write: 5759 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5760 mlxsw_sp->router->adj_discard_index); 5761 return err; 5762 } 5763 5764 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5765 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5766 struct mlxsw_sp_fib_entry *fib_entry, 5767 enum mlxsw_sp_fib_entry_op op) 5768 { 5769 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5770 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5771 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5772 enum mlxsw_reg_ralue_trap_action trap_action; 5773 u16 trap_id = 0; 5774 u32 adjacency_index = 0; 5775 u16 ecmp_size = 0; 5776 int err; 5777 5778 /* In case the nexthop group adjacency index is valid, use it 5779 * with provided ECMP size. Otherwise, setup trap and pass 5780 * traffic to kernel. 5781 */ 5782 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5783 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5784 adjacency_index = nhgi->adj_index; 5785 ecmp_size = nhgi->ecmp_size; 5786 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5787 err = mlxsw_sp_adj_discard_write(mlxsw_sp); 5788 if (err) 5789 return err; 5790 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5791 adjacency_index = mlxsw_sp->router->adj_discard_index; 5792 ecmp_size = 1; 5793 } else { 5794 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5795 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5796 } 5797 5798 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5799 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5800 adjacency_index, ecmp_size); 5801 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5802 } 5803 5804 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5805 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5806 struct mlxsw_sp_fib_entry *fib_entry, 5807 enum mlxsw_sp_fib_entry_op op) 5808 { 5809 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5810 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5811 enum mlxsw_reg_ralue_trap_action trap_action; 5812 u16 trap_id = 0; 5813 u16 rif_index = 0; 5814 5815 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5816 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5817 rif_index = rif->rif_index; 5818 } else { 5819 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5820 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5821 } 5822 5823 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5824 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5825 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5826 } 5827 5828 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5829 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5830 struct mlxsw_sp_fib_entry *fib_entry, 5831 enum mlxsw_sp_fib_entry_op op) 5832 { 5833 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5834 5835 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5836 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5837 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5838 } 5839 5840 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5841 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5842 struct mlxsw_sp_fib_entry *fib_entry, 5843 enum mlxsw_sp_fib_entry_op op) 5844 { 5845 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5846 enum mlxsw_reg_ralue_trap_action trap_action; 5847 5848 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5849 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5850 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 5851 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5852 } 5853 5854 static int 5855 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5856 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5857 struct mlxsw_sp_fib_entry *fib_entry, 5858 enum mlxsw_sp_fib_entry_op op) 5859 { 5860 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5861 enum mlxsw_reg_ralue_trap_action trap_action; 5862 u16 trap_id; 5863 5864 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5865 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5866 5867 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5868 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 5869 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5870 } 5871 5872 static int 5873 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5874 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5875 struct mlxsw_sp_fib_entry *fib_entry, 5876 enum mlxsw_sp_fib_entry_op op) 5877 { 5878 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5879 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5880 const struct mlxsw_sp_ipip_ops *ipip_ops; 5881 int err; 5882 5883 if (WARN_ON(!ipip_entry)) 5884 return -EINVAL; 5885 5886 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5887 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5888 fib_entry->decap.tunnel_index); 5889 if (err) 5890 return err; 5891 5892 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5893 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5894 fib_entry->decap.tunnel_index); 5895 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5896 } 5897 5898 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5899 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5900 struct mlxsw_sp_fib_entry *fib_entry, 5901 enum mlxsw_sp_fib_entry_op op) 5902 { 5903 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5904 5905 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5906 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5907 fib_entry->decap.tunnel_index); 5908 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5909 } 5910 5911 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5912 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5913 struct mlxsw_sp_fib_entry *fib_entry, 5914 enum mlxsw_sp_fib_entry_op op) 5915 { 5916 switch (fib_entry->type) { 5917 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5918 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 5919 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5920 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 5921 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5922 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 5923 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5924 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 5925 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5926 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 5927 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5928 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 5929 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5930 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 5931 } 5932 return -EINVAL; 5933 } 5934 5935 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5936 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5937 struct mlxsw_sp_fib_entry *fib_entry, 5938 enum mlxsw_sp_fib_entry_op op) 5939 { 5940 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 5941 5942 if (err) 5943 return err; 5944 5945 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5946 5947 return err; 5948 } 5949 5950 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5951 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5952 struct mlxsw_sp_fib_entry *fib_entry, 5953 bool is_new) 5954 { 5955 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5956 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 5957 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 5958 } 5959 5960 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5961 struct mlxsw_sp_fib_entry *fib_entry) 5962 { 5963 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5964 5965 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5966 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 5967 } 5968 5969 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5970 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5971 struct mlxsw_sp_fib_entry *fib_entry) 5972 { 5973 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5974 5975 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 5976 return 0; 5977 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5978 MLXSW_SP_FIB_ENTRY_OP_DELETE); 5979 } 5980 5981 static int 5982 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5983 const struct fib_entry_notifier_info *fen_info, 5984 struct mlxsw_sp_fib_entry *fib_entry) 5985 { 5986 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5987 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5988 struct mlxsw_sp_router *router = mlxsw_sp->router; 5989 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5990 int ifindex = nhgi->nexthops[0].ifindex; 5991 struct mlxsw_sp_ipip_entry *ipip_entry; 5992 5993 switch (fen_info->type) { 5994 case RTN_LOCAL: 5995 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5996 MLXSW_SP_L3_PROTO_IPV4, dip); 5997 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 5998 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 5999 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6000 fib_entry, 6001 ipip_entry); 6002 } 6003 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6004 MLXSW_SP_L3_PROTO_IPV4, 6005 &dip)) { 6006 u32 tunnel_index; 6007 6008 tunnel_index = router->nve_decap_config.tunnel_index; 6009 fib_entry->decap.tunnel_index = tunnel_index; 6010 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6011 return 0; 6012 } 6013 fallthrough; 6014 case RTN_BROADCAST: 6015 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6016 return 0; 6017 case RTN_BLACKHOLE: 6018 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6019 return 0; 6020 case RTN_UNREACHABLE: 6021 case RTN_PROHIBIT: 6022 /* Packets hitting these routes need to be trapped, but 6023 * can do so with a lower priority than packets directed 6024 * at the host, so use action type local instead of trap. 6025 */ 6026 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6027 return 0; 6028 case RTN_UNICAST: 6029 if (nhgi->gateway) 6030 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6031 else 6032 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6033 return 0; 6034 default: 6035 return -EINVAL; 6036 } 6037 } 6038 6039 static void 6040 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6041 struct mlxsw_sp_fib_entry *fib_entry) 6042 { 6043 switch (fib_entry->type) { 6044 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6045 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6046 break; 6047 default: 6048 break; 6049 } 6050 } 6051 6052 static struct mlxsw_sp_fib4_entry * 6053 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6054 struct mlxsw_sp_fib_node *fib_node, 6055 const struct fib_entry_notifier_info *fen_info) 6056 { 6057 struct mlxsw_sp_fib4_entry *fib4_entry; 6058 struct mlxsw_sp_fib_entry *fib_entry; 6059 int err; 6060 6061 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6062 if (!fib4_entry) 6063 return ERR_PTR(-ENOMEM); 6064 fib_entry = &fib4_entry->common; 6065 6066 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6067 if (IS_ERR(fib_entry->priv)) { 6068 err = PTR_ERR(fib_entry->priv); 6069 goto err_fib_entry_priv_create; 6070 } 6071 6072 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6073 if (err) 6074 goto err_nexthop4_group_get; 6075 6076 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6077 fib_node->fib); 6078 if (err) 6079 goto err_nexthop_group_vr_link; 6080 6081 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6082 if (err) 6083 goto err_fib4_entry_type_set; 6084 6085 fib4_entry->fi = fen_info->fi; 6086 fib_info_hold(fib4_entry->fi); 6087 fib4_entry->tb_id = fen_info->tb_id; 6088 fib4_entry->type = fen_info->type; 6089 fib4_entry->tos = fen_info->tos; 6090 6091 fib_entry->fib_node = fib_node; 6092 6093 return fib4_entry; 6094 6095 err_fib4_entry_type_set: 6096 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6097 err_nexthop_group_vr_link: 6098 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6099 err_nexthop4_group_get: 6100 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6101 err_fib_entry_priv_create: 6102 kfree(fib4_entry); 6103 return ERR_PTR(err); 6104 } 6105 6106 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6107 struct mlxsw_sp_fib4_entry *fib4_entry) 6108 { 6109 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6110 6111 fib_info_put(fib4_entry->fi); 6112 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6113 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6114 fib_node->fib); 6115 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6116 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 6117 kfree(fib4_entry); 6118 } 6119 6120 static struct mlxsw_sp_fib4_entry * 6121 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6122 const struct fib_entry_notifier_info *fen_info) 6123 { 6124 struct mlxsw_sp_fib4_entry *fib4_entry; 6125 struct mlxsw_sp_fib_node *fib_node; 6126 struct mlxsw_sp_fib *fib; 6127 struct mlxsw_sp_vr *vr; 6128 6129 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6130 if (!vr) 6131 return NULL; 6132 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6133 6134 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6135 sizeof(fen_info->dst), 6136 fen_info->dst_len); 6137 if (!fib_node) 6138 return NULL; 6139 6140 fib4_entry = container_of(fib_node->fib_entry, 6141 struct mlxsw_sp_fib4_entry, common); 6142 if (fib4_entry->tb_id == fen_info->tb_id && 6143 fib4_entry->tos == fen_info->tos && 6144 fib4_entry->type == fen_info->type && 6145 fib4_entry->fi == fen_info->fi) 6146 return fib4_entry; 6147 6148 return NULL; 6149 } 6150 6151 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6152 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6153 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6154 .key_len = sizeof(struct mlxsw_sp_fib_key), 6155 .automatic_shrinking = true, 6156 }; 6157 6158 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6159 struct mlxsw_sp_fib_node *fib_node) 6160 { 6161 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6162 mlxsw_sp_fib_ht_params); 6163 } 6164 6165 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6166 struct mlxsw_sp_fib_node *fib_node) 6167 { 6168 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6169 mlxsw_sp_fib_ht_params); 6170 } 6171 6172 static struct mlxsw_sp_fib_node * 6173 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6174 size_t addr_len, unsigned char prefix_len) 6175 { 6176 struct mlxsw_sp_fib_key key; 6177 6178 memset(&key, 0, sizeof(key)); 6179 memcpy(key.addr, addr, addr_len); 6180 key.prefix_len = prefix_len; 6181 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6182 } 6183 6184 static struct mlxsw_sp_fib_node * 6185 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6186 size_t addr_len, unsigned char prefix_len) 6187 { 6188 struct mlxsw_sp_fib_node *fib_node; 6189 6190 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6191 if (!fib_node) 6192 return NULL; 6193 6194 list_add(&fib_node->list, &fib->node_list); 6195 memcpy(fib_node->key.addr, addr, addr_len); 6196 fib_node->key.prefix_len = prefix_len; 6197 6198 return fib_node; 6199 } 6200 6201 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6202 { 6203 list_del(&fib_node->list); 6204 kfree(fib_node); 6205 } 6206 6207 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6208 struct mlxsw_sp_fib_node *fib_node) 6209 { 6210 struct mlxsw_sp_prefix_usage req_prefix_usage; 6211 struct mlxsw_sp_fib *fib = fib_node->fib; 6212 struct mlxsw_sp_lpm_tree *lpm_tree; 6213 int err; 6214 6215 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6216 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6217 goto out; 6218 6219 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6220 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6221 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6222 fib->proto); 6223 if (IS_ERR(lpm_tree)) 6224 return PTR_ERR(lpm_tree); 6225 6226 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6227 if (err) 6228 goto err_lpm_tree_replace; 6229 6230 out: 6231 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6232 return 0; 6233 6234 err_lpm_tree_replace: 6235 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6236 return err; 6237 } 6238 6239 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6240 struct mlxsw_sp_fib_node *fib_node) 6241 { 6242 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6243 struct mlxsw_sp_prefix_usage req_prefix_usage; 6244 struct mlxsw_sp_fib *fib = fib_node->fib; 6245 int err; 6246 6247 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6248 return; 6249 /* Try to construct a new LPM tree from the current prefix usage 6250 * minus the unused one. If we fail, continue using the old one. 6251 */ 6252 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6253 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6254 fib_node->key.prefix_len); 6255 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6256 fib->proto); 6257 if (IS_ERR(lpm_tree)) 6258 return; 6259 6260 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6261 if (err) 6262 goto err_lpm_tree_replace; 6263 6264 return; 6265 6266 err_lpm_tree_replace: 6267 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6268 } 6269 6270 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6271 struct mlxsw_sp_fib_node *fib_node, 6272 struct mlxsw_sp_fib *fib) 6273 { 6274 int err; 6275 6276 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6277 if (err) 6278 return err; 6279 fib_node->fib = fib; 6280 6281 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6282 if (err) 6283 goto err_fib_lpm_tree_link; 6284 6285 return 0; 6286 6287 err_fib_lpm_tree_link: 6288 fib_node->fib = NULL; 6289 mlxsw_sp_fib_node_remove(fib, fib_node); 6290 return err; 6291 } 6292 6293 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6294 struct mlxsw_sp_fib_node *fib_node) 6295 { 6296 struct mlxsw_sp_fib *fib = fib_node->fib; 6297 6298 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6299 fib_node->fib = NULL; 6300 mlxsw_sp_fib_node_remove(fib, fib_node); 6301 } 6302 6303 static struct mlxsw_sp_fib_node * 6304 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6305 size_t addr_len, unsigned char prefix_len, 6306 enum mlxsw_sp_l3proto proto) 6307 { 6308 struct mlxsw_sp_fib_node *fib_node; 6309 struct mlxsw_sp_fib *fib; 6310 struct mlxsw_sp_vr *vr; 6311 int err; 6312 6313 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6314 if (IS_ERR(vr)) 6315 return ERR_CAST(vr); 6316 fib = mlxsw_sp_vr_fib(vr, proto); 6317 6318 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6319 if (fib_node) 6320 return fib_node; 6321 6322 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6323 if (!fib_node) { 6324 err = -ENOMEM; 6325 goto err_fib_node_create; 6326 } 6327 6328 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6329 if (err) 6330 goto err_fib_node_init; 6331 6332 return fib_node; 6333 6334 err_fib_node_init: 6335 mlxsw_sp_fib_node_destroy(fib_node); 6336 err_fib_node_create: 6337 mlxsw_sp_vr_put(mlxsw_sp, vr); 6338 return ERR_PTR(err); 6339 } 6340 6341 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6342 struct mlxsw_sp_fib_node *fib_node) 6343 { 6344 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6345 6346 if (fib_node->fib_entry) 6347 return; 6348 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6349 mlxsw_sp_fib_node_destroy(fib_node); 6350 mlxsw_sp_vr_put(mlxsw_sp, vr); 6351 } 6352 6353 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6354 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6355 struct mlxsw_sp_fib_entry *fib_entry) 6356 { 6357 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6358 bool is_new = !fib_node->fib_entry; 6359 int err; 6360 6361 fib_node->fib_entry = fib_entry; 6362 6363 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 6364 if (err) 6365 goto err_fib_entry_update; 6366 6367 return 0; 6368 6369 err_fib_entry_update: 6370 fib_node->fib_entry = NULL; 6371 return err; 6372 } 6373 6374 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6375 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6376 struct mlxsw_sp_fib_entry *fib_entry) 6377 { 6378 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6379 int err; 6380 6381 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 6382 fib_node->fib_entry = NULL; 6383 return err; 6384 } 6385 6386 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6387 struct mlxsw_sp_fib_entry *fib_entry) 6388 { 6389 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6390 6391 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6392 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 6393 } 6394 6395 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6396 { 6397 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6398 struct mlxsw_sp_fib4_entry *fib4_replaced; 6399 6400 if (!fib_node->fib_entry) 6401 return true; 6402 6403 fib4_replaced = container_of(fib_node->fib_entry, 6404 struct mlxsw_sp_fib4_entry, common); 6405 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6406 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6407 return false; 6408 6409 return true; 6410 } 6411 6412 static int 6413 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6414 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6415 const struct fib_entry_notifier_info *fen_info) 6416 { 6417 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6418 struct mlxsw_sp_fib_entry *replaced; 6419 struct mlxsw_sp_fib_node *fib_node; 6420 int err; 6421 6422 if (fen_info->fi->nh && 6423 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6424 return 0; 6425 6426 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6427 &fen_info->dst, sizeof(fen_info->dst), 6428 fen_info->dst_len, 6429 MLXSW_SP_L3_PROTO_IPV4); 6430 if (IS_ERR(fib_node)) { 6431 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6432 return PTR_ERR(fib_node); 6433 } 6434 6435 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6436 if (IS_ERR(fib4_entry)) { 6437 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6438 err = PTR_ERR(fib4_entry); 6439 goto err_fib4_entry_create; 6440 } 6441 6442 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6443 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6444 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6445 return 0; 6446 } 6447 6448 replaced = fib_node->fib_entry; 6449 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 6450 if (err) { 6451 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6452 goto err_fib_node_entry_link; 6453 } 6454 6455 /* Nothing to replace */ 6456 if (!replaced) 6457 return 0; 6458 6459 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6460 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6461 common); 6462 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6463 6464 return 0; 6465 6466 err_fib_node_entry_link: 6467 fib_node->fib_entry = replaced; 6468 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6469 err_fib4_entry_create: 6470 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6471 return err; 6472 } 6473 6474 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6475 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6476 struct fib_entry_notifier_info *fen_info) 6477 { 6478 struct mlxsw_sp_fib4_entry *fib4_entry; 6479 struct mlxsw_sp_fib_node *fib_node; 6480 int err; 6481 6482 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6483 if (!fib4_entry) 6484 return 0; 6485 fib_node = fib4_entry->common.fib_node; 6486 6487 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6488 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6489 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6490 return err; 6491 } 6492 6493 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6494 { 6495 /* Multicast routes aren't supported, so ignore them. Neighbour 6496 * Discovery packets are specifically trapped. 6497 */ 6498 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6499 return true; 6500 6501 /* Cloned routes are irrelevant in the forwarding path. */ 6502 if (rt->fib6_flags & RTF_CACHE) 6503 return true; 6504 6505 return false; 6506 } 6507 6508 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6509 { 6510 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6511 6512 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6513 if (!mlxsw_sp_rt6) 6514 return ERR_PTR(-ENOMEM); 6515 6516 /* In case of route replace, replaced route is deleted with 6517 * no notification. Take reference to prevent accessing freed 6518 * memory. 6519 */ 6520 mlxsw_sp_rt6->rt = rt; 6521 fib6_info_hold(rt); 6522 6523 return mlxsw_sp_rt6; 6524 } 6525 6526 #if IS_ENABLED(CONFIG_IPV6) 6527 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6528 { 6529 fib6_info_release(rt); 6530 } 6531 #else 6532 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6533 { 6534 } 6535 #endif 6536 6537 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6538 { 6539 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6540 6541 if (!mlxsw_sp_rt6->rt->nh) 6542 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6543 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6544 kfree(mlxsw_sp_rt6); 6545 } 6546 6547 static struct fib6_info * 6548 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6549 { 6550 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6551 list)->rt; 6552 } 6553 6554 static struct mlxsw_sp_rt6 * 6555 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6556 const struct fib6_info *rt) 6557 { 6558 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6559 6560 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6561 if (mlxsw_sp_rt6->rt == rt) 6562 return mlxsw_sp_rt6; 6563 } 6564 6565 return NULL; 6566 } 6567 6568 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6569 const struct fib6_info *rt, 6570 enum mlxsw_sp_ipip_type *ret) 6571 { 6572 return rt->fib6_nh->fib_nh_dev && 6573 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6574 } 6575 6576 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6577 struct mlxsw_sp_nexthop_group *nh_grp, 6578 struct mlxsw_sp_nexthop *nh, 6579 const struct fib6_info *rt) 6580 { 6581 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6582 6583 nh->nhgi = nh_grp->nhgi; 6584 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6585 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6586 #if IS_ENABLED(CONFIG_IPV6) 6587 nh->neigh_tbl = &nd_tbl; 6588 #endif 6589 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6590 6591 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6592 6593 if (!dev) 6594 return 0; 6595 nh->ifindex = dev->ifindex; 6596 6597 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6598 } 6599 6600 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6601 struct mlxsw_sp_nexthop *nh) 6602 { 6603 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6604 list_del(&nh->router_list_node); 6605 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6606 } 6607 6608 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6609 const struct fib6_info *rt) 6610 { 6611 return rt->fib6_nh->fib_nh_gw_family || 6612 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6613 } 6614 6615 static int 6616 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6617 struct mlxsw_sp_nexthop_group *nh_grp, 6618 struct mlxsw_sp_fib6_entry *fib6_entry) 6619 { 6620 struct mlxsw_sp_nexthop_group_info *nhgi; 6621 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6622 struct mlxsw_sp_nexthop *nh; 6623 int err, i; 6624 6625 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6626 GFP_KERNEL); 6627 if (!nhgi) 6628 return -ENOMEM; 6629 nh_grp->nhgi = nhgi; 6630 nhgi->nh_grp = nh_grp; 6631 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6632 struct mlxsw_sp_rt6, list); 6633 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6634 nhgi->count = fib6_entry->nrt6; 6635 for (i = 0; i < nhgi->count; i++) { 6636 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6637 6638 nh = &nhgi->nexthops[i]; 6639 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6640 if (err) 6641 goto err_nexthop6_init; 6642 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6643 } 6644 nh_grp->nhgi = nhgi; 6645 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6646 if (err) 6647 goto err_group_refresh; 6648 6649 return 0; 6650 6651 err_group_refresh: 6652 i = nhgi->count; 6653 err_nexthop6_init: 6654 for (i--; i >= 0; i--) { 6655 nh = &nhgi->nexthops[i]; 6656 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6657 } 6658 kfree(nhgi); 6659 return err; 6660 } 6661 6662 static void 6663 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6664 struct mlxsw_sp_nexthop_group *nh_grp) 6665 { 6666 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6667 int i; 6668 6669 for (i = nhgi->count - 1; i >= 0; i--) { 6670 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6671 6672 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6673 } 6674 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6675 WARN_ON_ONCE(nhgi->adj_index_valid); 6676 kfree(nhgi); 6677 } 6678 6679 static struct mlxsw_sp_nexthop_group * 6680 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6681 struct mlxsw_sp_fib6_entry *fib6_entry) 6682 { 6683 struct mlxsw_sp_nexthop_group *nh_grp; 6684 int err; 6685 6686 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6687 if (!nh_grp) 6688 return ERR_PTR(-ENOMEM); 6689 INIT_LIST_HEAD(&nh_grp->vr_list); 6690 err = rhashtable_init(&nh_grp->vr_ht, 6691 &mlxsw_sp_nexthop_group_vr_ht_params); 6692 if (err) 6693 goto err_nexthop_group_vr_ht_init; 6694 INIT_LIST_HEAD(&nh_grp->fib_list); 6695 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6696 6697 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6698 if (err) 6699 goto err_nexthop_group_info_init; 6700 6701 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6702 if (err) 6703 goto err_nexthop_group_insert; 6704 6705 nh_grp->can_destroy = true; 6706 6707 return nh_grp; 6708 6709 err_nexthop_group_insert: 6710 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6711 err_nexthop_group_info_init: 6712 rhashtable_destroy(&nh_grp->vr_ht); 6713 err_nexthop_group_vr_ht_init: 6714 kfree(nh_grp); 6715 return ERR_PTR(err); 6716 } 6717 6718 static void 6719 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6720 struct mlxsw_sp_nexthop_group *nh_grp) 6721 { 6722 if (!nh_grp->can_destroy) 6723 return; 6724 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6725 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6726 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6727 rhashtable_destroy(&nh_grp->vr_ht); 6728 kfree(nh_grp); 6729 } 6730 6731 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6732 struct mlxsw_sp_fib6_entry *fib6_entry) 6733 { 6734 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6735 struct mlxsw_sp_nexthop_group *nh_grp; 6736 6737 if (rt->nh) { 6738 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6739 rt->nh->id); 6740 if (WARN_ON_ONCE(!nh_grp)) 6741 return -EINVAL; 6742 goto out; 6743 } 6744 6745 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6746 if (!nh_grp) { 6747 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6748 if (IS_ERR(nh_grp)) 6749 return PTR_ERR(nh_grp); 6750 } 6751 6752 /* The route and the nexthop are described by the same struct, so we 6753 * need to the update the nexthop offload indication for the new route. 6754 */ 6755 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6756 6757 out: 6758 list_add_tail(&fib6_entry->common.nexthop_group_node, 6759 &nh_grp->fib_list); 6760 fib6_entry->common.nh_group = nh_grp; 6761 6762 return 0; 6763 } 6764 6765 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6766 struct mlxsw_sp_fib_entry *fib_entry) 6767 { 6768 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6769 6770 list_del(&fib_entry->nexthop_group_node); 6771 if (!list_empty(&nh_grp->fib_list)) 6772 return; 6773 6774 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6775 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6776 return; 6777 } 6778 6779 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6780 } 6781 6782 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6783 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6784 struct mlxsw_sp_fib6_entry *fib6_entry) 6785 { 6786 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6787 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6788 int err; 6789 6790 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6791 fib6_entry->common.nh_group = NULL; 6792 list_del(&fib6_entry->common.nexthop_group_node); 6793 6794 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6795 if (err) 6796 goto err_nexthop6_group_get; 6797 6798 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6799 fib_node->fib); 6800 if (err) 6801 goto err_nexthop_group_vr_link; 6802 6803 /* In case this entry is offloaded, then the adjacency index 6804 * currently associated with it in the device's table is that 6805 * of the old group. Start using the new one instead. 6806 */ 6807 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6808 &fib6_entry->common, false); 6809 if (err) 6810 goto err_fib_entry_update; 6811 6812 if (list_empty(&old_nh_grp->fib_list)) 6813 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6814 6815 return 0; 6816 6817 err_fib_entry_update: 6818 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6819 fib_node->fib); 6820 err_nexthop_group_vr_link: 6821 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6822 err_nexthop6_group_get: 6823 list_add_tail(&fib6_entry->common.nexthop_group_node, 6824 &old_nh_grp->fib_list); 6825 fib6_entry->common.nh_group = old_nh_grp; 6826 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6827 return err; 6828 } 6829 6830 static int 6831 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6832 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6833 struct mlxsw_sp_fib6_entry *fib6_entry, 6834 struct fib6_info **rt_arr, unsigned int nrt6) 6835 { 6836 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6837 int err, i; 6838 6839 for (i = 0; i < nrt6; i++) { 6840 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6841 if (IS_ERR(mlxsw_sp_rt6)) { 6842 err = PTR_ERR(mlxsw_sp_rt6); 6843 goto err_rt6_create; 6844 } 6845 6846 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6847 fib6_entry->nrt6++; 6848 } 6849 6850 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6851 if (err) 6852 goto err_nexthop6_group_update; 6853 6854 return 0; 6855 6856 err_nexthop6_group_update: 6857 i = nrt6; 6858 err_rt6_create: 6859 for (i--; i >= 0; i--) { 6860 fib6_entry->nrt6--; 6861 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6862 struct mlxsw_sp_rt6, list); 6863 list_del(&mlxsw_sp_rt6->list); 6864 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6865 } 6866 return err; 6867 } 6868 6869 static void 6870 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6871 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6872 struct mlxsw_sp_fib6_entry *fib6_entry, 6873 struct fib6_info **rt_arr, unsigned int nrt6) 6874 { 6875 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6876 int i; 6877 6878 for (i = 0; i < nrt6; i++) { 6879 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6880 rt_arr[i]); 6881 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6882 continue; 6883 6884 fib6_entry->nrt6--; 6885 list_del(&mlxsw_sp_rt6->list); 6886 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6887 } 6888 6889 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6890 } 6891 6892 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6893 struct mlxsw_sp_fib_entry *fib_entry, 6894 const struct fib6_info *rt) 6895 { 6896 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 6897 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6898 else if (rt->fib6_type == RTN_BLACKHOLE) 6899 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6900 else if (rt->fib6_flags & RTF_REJECT) 6901 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6902 else if (fib_entry->nh_group->nhgi->gateway) 6903 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6904 else 6905 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6906 } 6907 6908 static void 6909 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6910 { 6911 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6912 6913 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6914 list) { 6915 fib6_entry->nrt6--; 6916 list_del(&mlxsw_sp_rt6->list); 6917 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6918 } 6919 } 6920 6921 static struct mlxsw_sp_fib6_entry * 6922 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6923 struct mlxsw_sp_fib_node *fib_node, 6924 struct fib6_info **rt_arr, unsigned int nrt6) 6925 { 6926 struct mlxsw_sp_fib6_entry *fib6_entry; 6927 struct mlxsw_sp_fib_entry *fib_entry; 6928 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6929 int err, i; 6930 6931 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6932 if (!fib6_entry) 6933 return ERR_PTR(-ENOMEM); 6934 fib_entry = &fib6_entry->common; 6935 6936 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6937 if (IS_ERR(fib_entry->priv)) { 6938 err = PTR_ERR(fib_entry->priv); 6939 goto err_fib_entry_priv_create; 6940 } 6941 6942 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6943 6944 for (i = 0; i < nrt6; i++) { 6945 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6946 if (IS_ERR(mlxsw_sp_rt6)) { 6947 err = PTR_ERR(mlxsw_sp_rt6); 6948 goto err_rt6_create; 6949 } 6950 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6951 fib6_entry->nrt6++; 6952 } 6953 6954 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6955 if (err) 6956 goto err_nexthop6_group_get; 6957 6958 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6959 fib_node->fib); 6960 if (err) 6961 goto err_nexthop_group_vr_link; 6962 6963 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6964 6965 fib_entry->fib_node = fib_node; 6966 6967 return fib6_entry; 6968 6969 err_nexthop_group_vr_link: 6970 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6971 err_nexthop6_group_get: 6972 i = nrt6; 6973 err_rt6_create: 6974 for (i--; i >= 0; i--) { 6975 fib6_entry->nrt6--; 6976 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6977 struct mlxsw_sp_rt6, list); 6978 list_del(&mlxsw_sp_rt6->list); 6979 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6980 } 6981 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6982 err_fib_entry_priv_create: 6983 kfree(fib6_entry); 6984 return ERR_PTR(err); 6985 } 6986 6987 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6988 struct mlxsw_sp_fib6_entry *fib6_entry) 6989 { 6990 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6991 6992 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6993 fib_node->fib); 6994 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6995 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 6996 WARN_ON(fib6_entry->nrt6); 6997 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 6998 kfree(fib6_entry); 6999 } 7000 7001 static struct mlxsw_sp_fib6_entry * 7002 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7003 const struct fib6_info *rt) 7004 { 7005 struct mlxsw_sp_fib6_entry *fib6_entry; 7006 struct mlxsw_sp_fib_node *fib_node; 7007 struct mlxsw_sp_fib *fib; 7008 struct fib6_info *cmp_rt; 7009 struct mlxsw_sp_vr *vr; 7010 7011 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7012 if (!vr) 7013 return NULL; 7014 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7015 7016 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7017 sizeof(rt->fib6_dst.addr), 7018 rt->fib6_dst.plen); 7019 if (!fib_node) 7020 return NULL; 7021 7022 fib6_entry = container_of(fib_node->fib_entry, 7023 struct mlxsw_sp_fib6_entry, common); 7024 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7025 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7026 rt->fib6_metric == cmp_rt->fib6_metric && 7027 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7028 return fib6_entry; 7029 7030 return NULL; 7031 } 7032 7033 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7034 { 7035 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7036 struct mlxsw_sp_fib6_entry *fib6_replaced; 7037 struct fib6_info *rt, *rt_replaced; 7038 7039 if (!fib_node->fib_entry) 7040 return true; 7041 7042 fib6_replaced = container_of(fib_node->fib_entry, 7043 struct mlxsw_sp_fib6_entry, 7044 common); 7045 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7046 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7047 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7048 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7049 return false; 7050 7051 return true; 7052 } 7053 7054 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7055 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7056 struct fib6_info **rt_arr, unsigned int nrt6) 7057 { 7058 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7059 struct mlxsw_sp_fib_entry *replaced; 7060 struct mlxsw_sp_fib_node *fib_node; 7061 struct fib6_info *rt = rt_arr[0]; 7062 int err; 7063 7064 if (rt->fib6_src.plen) 7065 return -EINVAL; 7066 7067 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7068 return 0; 7069 7070 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7071 return 0; 7072 7073 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7074 &rt->fib6_dst.addr, 7075 sizeof(rt->fib6_dst.addr), 7076 rt->fib6_dst.plen, 7077 MLXSW_SP_L3_PROTO_IPV6); 7078 if (IS_ERR(fib_node)) 7079 return PTR_ERR(fib_node); 7080 7081 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7082 nrt6); 7083 if (IS_ERR(fib6_entry)) { 7084 err = PTR_ERR(fib6_entry); 7085 goto err_fib6_entry_create; 7086 } 7087 7088 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7089 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7090 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7091 return 0; 7092 } 7093 7094 replaced = fib_node->fib_entry; 7095 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 7096 if (err) 7097 goto err_fib_node_entry_link; 7098 7099 /* Nothing to replace */ 7100 if (!replaced) 7101 return 0; 7102 7103 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7104 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7105 common); 7106 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7107 7108 return 0; 7109 7110 err_fib_node_entry_link: 7111 fib_node->fib_entry = replaced; 7112 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7113 err_fib6_entry_create: 7114 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7115 return err; 7116 } 7117 7118 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7119 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7120 struct fib6_info **rt_arr, unsigned int nrt6) 7121 { 7122 struct mlxsw_sp_fib6_entry *fib6_entry; 7123 struct mlxsw_sp_fib_node *fib_node; 7124 struct fib6_info *rt = rt_arr[0]; 7125 int err; 7126 7127 if (rt->fib6_src.plen) 7128 return -EINVAL; 7129 7130 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7131 return 0; 7132 7133 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7134 &rt->fib6_dst.addr, 7135 sizeof(rt->fib6_dst.addr), 7136 rt->fib6_dst.plen, 7137 MLXSW_SP_L3_PROTO_IPV6); 7138 if (IS_ERR(fib_node)) 7139 return PTR_ERR(fib_node); 7140 7141 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7142 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7143 return -EINVAL; 7144 } 7145 7146 fib6_entry = container_of(fib_node->fib_entry, 7147 struct mlxsw_sp_fib6_entry, common); 7148 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7149 if (err) 7150 goto err_fib6_entry_nexthop_add; 7151 7152 return 0; 7153 7154 err_fib6_entry_nexthop_add: 7155 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7156 return err; 7157 } 7158 7159 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7160 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7161 struct fib6_info **rt_arr, unsigned int nrt6) 7162 { 7163 struct mlxsw_sp_fib6_entry *fib6_entry; 7164 struct mlxsw_sp_fib_node *fib_node; 7165 struct fib6_info *rt = rt_arr[0]; 7166 int err; 7167 7168 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7169 return 0; 7170 7171 /* Multipath routes are first added to the FIB trie and only then 7172 * notified. If we vetoed the addition, we will get a delete 7173 * notification for a route we do not have. Therefore, do not warn if 7174 * route was not found. 7175 */ 7176 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7177 if (!fib6_entry) 7178 return 0; 7179 7180 /* If not all the nexthops are deleted, then only reduce the nexthop 7181 * group. 7182 */ 7183 if (nrt6 != fib6_entry->nrt6) { 7184 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7185 return 0; 7186 } 7187 7188 fib_node = fib6_entry->common.fib_node; 7189 7190 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 7191 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7192 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7193 return err; 7194 } 7195 7196 static struct mlxsw_sp_mr_table * 7197 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7198 { 7199 if (family == RTNL_FAMILY_IPMR) 7200 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7201 else 7202 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7203 } 7204 7205 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7206 struct mfc_entry_notifier_info *men_info, 7207 bool replace) 7208 { 7209 struct mlxsw_sp_mr_table *mrt; 7210 struct mlxsw_sp_vr *vr; 7211 7212 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7213 if (IS_ERR(vr)) 7214 return PTR_ERR(vr); 7215 7216 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7217 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7218 } 7219 7220 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7221 struct mfc_entry_notifier_info *men_info) 7222 { 7223 struct mlxsw_sp_mr_table *mrt; 7224 struct mlxsw_sp_vr *vr; 7225 7226 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7227 if (WARN_ON(!vr)) 7228 return; 7229 7230 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7231 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7232 mlxsw_sp_vr_put(mlxsw_sp, vr); 7233 } 7234 7235 static int 7236 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7237 struct vif_entry_notifier_info *ven_info) 7238 { 7239 struct mlxsw_sp_mr_table *mrt; 7240 struct mlxsw_sp_rif *rif; 7241 struct mlxsw_sp_vr *vr; 7242 7243 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7244 if (IS_ERR(vr)) 7245 return PTR_ERR(vr); 7246 7247 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7248 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7249 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7250 ven_info->vif_index, 7251 ven_info->vif_flags, rif); 7252 } 7253 7254 static void 7255 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7256 struct vif_entry_notifier_info *ven_info) 7257 { 7258 struct mlxsw_sp_mr_table *mrt; 7259 struct mlxsw_sp_vr *vr; 7260 7261 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7262 if (WARN_ON(!vr)) 7263 return; 7264 7265 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7266 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7267 mlxsw_sp_vr_put(mlxsw_sp, vr); 7268 } 7269 7270 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7271 struct mlxsw_sp_fib_node *fib_node) 7272 { 7273 struct mlxsw_sp_fib4_entry *fib4_entry; 7274 7275 fib4_entry = container_of(fib_node->fib_entry, 7276 struct mlxsw_sp_fib4_entry, common); 7277 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7278 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7279 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7280 } 7281 7282 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7283 struct mlxsw_sp_fib_node *fib_node) 7284 { 7285 struct mlxsw_sp_fib6_entry *fib6_entry; 7286 7287 fib6_entry = container_of(fib_node->fib_entry, 7288 struct mlxsw_sp_fib6_entry, common); 7289 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7290 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7291 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7292 } 7293 7294 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7295 struct mlxsw_sp_fib_node *fib_node) 7296 { 7297 switch (fib_node->fib->proto) { 7298 case MLXSW_SP_L3_PROTO_IPV4: 7299 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7300 break; 7301 case MLXSW_SP_L3_PROTO_IPV6: 7302 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7303 break; 7304 } 7305 } 7306 7307 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7308 struct mlxsw_sp_vr *vr, 7309 enum mlxsw_sp_l3proto proto) 7310 { 7311 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7312 struct mlxsw_sp_fib_node *fib_node, *tmp; 7313 7314 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7315 bool do_break = &tmp->list == &fib->node_list; 7316 7317 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7318 if (do_break) 7319 break; 7320 } 7321 } 7322 7323 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7324 { 7325 int i, j; 7326 7327 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7328 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7329 7330 if (!mlxsw_sp_vr_is_used(vr)) 7331 continue; 7332 7333 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7334 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7335 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7336 7337 /* If virtual router was only used for IPv4, then it's no 7338 * longer used. 7339 */ 7340 if (!mlxsw_sp_vr_is_used(vr)) 7341 continue; 7342 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7343 } 7344 7345 /* After flushing all the routes, it is not possible anyone is still 7346 * using the adjacency index that is discarding packets, so free it in 7347 * case it was allocated. 7348 */ 7349 if (!mlxsw_sp->router->adj_discard_index_valid) 7350 return; 7351 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 7352 mlxsw_sp->router->adj_discard_index); 7353 mlxsw_sp->router->adj_discard_index_valid = false; 7354 } 7355 7356 struct mlxsw_sp_fib6_event { 7357 struct fib6_info **rt_arr; 7358 unsigned int nrt6; 7359 }; 7360 7361 struct mlxsw_sp_fib_event { 7362 struct list_head list; /* node in fib queue */ 7363 union { 7364 struct mlxsw_sp_fib6_event fib6_event; 7365 struct fib_entry_notifier_info fen_info; 7366 struct fib_rule_notifier_info fr_info; 7367 struct fib_nh_notifier_info fnh_info; 7368 struct mfc_entry_notifier_info men_info; 7369 struct vif_entry_notifier_info ven_info; 7370 }; 7371 struct mlxsw_sp *mlxsw_sp; 7372 unsigned long event; 7373 int family; 7374 }; 7375 7376 static int 7377 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7378 struct fib6_entry_notifier_info *fen6_info) 7379 { 7380 struct fib6_info *rt = fen6_info->rt; 7381 struct fib6_info **rt_arr; 7382 struct fib6_info *iter; 7383 unsigned int nrt6; 7384 int i = 0; 7385 7386 nrt6 = fen6_info->nsiblings + 1; 7387 7388 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7389 if (!rt_arr) 7390 return -ENOMEM; 7391 7392 fib6_event->rt_arr = rt_arr; 7393 fib6_event->nrt6 = nrt6; 7394 7395 rt_arr[0] = rt; 7396 fib6_info_hold(rt); 7397 7398 if (!fen6_info->nsiblings) 7399 return 0; 7400 7401 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7402 if (i == fen6_info->nsiblings) 7403 break; 7404 7405 rt_arr[i + 1] = iter; 7406 fib6_info_hold(iter); 7407 i++; 7408 } 7409 WARN_ON_ONCE(i != fen6_info->nsiblings); 7410 7411 return 0; 7412 } 7413 7414 static void 7415 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7416 { 7417 int i; 7418 7419 for (i = 0; i < fib6_event->nrt6; i++) 7420 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7421 kfree(fib6_event->rt_arr); 7422 } 7423 7424 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7425 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7426 struct mlxsw_sp_fib_event *fib_event) 7427 { 7428 int err; 7429 7430 mlxsw_sp_span_respin(mlxsw_sp); 7431 7432 switch (fib_event->event) { 7433 case FIB_EVENT_ENTRY_REPLACE: 7434 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7435 if (err) { 7436 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7437 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7438 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7439 &fib_event->fen_info); 7440 } 7441 fib_info_put(fib_event->fen_info.fi); 7442 break; 7443 case FIB_EVENT_ENTRY_DEL: 7444 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7445 if (err) 7446 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7447 fib_info_put(fib_event->fen_info.fi); 7448 break; 7449 case FIB_EVENT_NH_ADD: 7450 case FIB_EVENT_NH_DEL: 7451 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7452 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7453 break; 7454 } 7455 } 7456 7457 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7458 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7459 struct mlxsw_sp_fib_event *fib_event) 7460 { 7461 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7462 int err; 7463 7464 mlxsw_sp_span_respin(mlxsw_sp); 7465 7466 switch (fib_event->event) { 7467 case FIB_EVENT_ENTRY_REPLACE: 7468 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7469 fib_event->fib6_event.nrt6); 7470 if (err) { 7471 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7472 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7473 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7474 fib6_event->rt_arr, 7475 fib6_event->nrt6); 7476 } 7477 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7478 break; 7479 case FIB_EVENT_ENTRY_APPEND: 7480 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7481 fib_event->fib6_event.nrt6); 7482 if (err) { 7483 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7484 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7485 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7486 fib6_event->rt_arr, 7487 fib6_event->nrt6); 7488 } 7489 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7490 break; 7491 case FIB_EVENT_ENTRY_DEL: 7492 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7493 fib_event->fib6_event.nrt6); 7494 if (err) 7495 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7496 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7497 break; 7498 } 7499 } 7500 7501 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7502 struct mlxsw_sp_fib_event *fib_event) 7503 { 7504 bool replace; 7505 int err; 7506 7507 rtnl_lock(); 7508 mutex_lock(&mlxsw_sp->router->lock); 7509 switch (fib_event->event) { 7510 case FIB_EVENT_ENTRY_REPLACE: 7511 case FIB_EVENT_ENTRY_ADD: 7512 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7513 7514 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7515 if (err) 7516 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7517 mr_cache_put(fib_event->men_info.mfc); 7518 break; 7519 case FIB_EVENT_ENTRY_DEL: 7520 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7521 mr_cache_put(fib_event->men_info.mfc); 7522 break; 7523 case FIB_EVENT_VIF_ADD: 7524 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7525 &fib_event->ven_info); 7526 if (err) 7527 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7528 dev_put(fib_event->ven_info.dev); 7529 break; 7530 case FIB_EVENT_VIF_DEL: 7531 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7532 dev_put(fib_event->ven_info.dev); 7533 break; 7534 } 7535 mutex_unlock(&mlxsw_sp->router->lock); 7536 rtnl_unlock(); 7537 } 7538 7539 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7540 { 7541 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7542 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7543 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7544 struct mlxsw_sp_fib_event *next_fib_event; 7545 struct mlxsw_sp_fib_event *fib_event; 7546 int last_family = AF_UNSPEC; 7547 LIST_HEAD(fib_event_queue); 7548 7549 spin_lock_bh(&router->fib_event_queue_lock); 7550 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7551 spin_unlock_bh(&router->fib_event_queue_lock); 7552 7553 /* Router lock is held here to make sure per-instance 7554 * operation context is not used in between FIB4/6 events 7555 * processing. 7556 */ 7557 mutex_lock(&router->lock); 7558 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7559 list_for_each_entry_safe(fib_event, next_fib_event, 7560 &fib_event_queue, list) { 7561 /* Check if the next entry in the queue exists and it is 7562 * of the same type (family and event) as the currect one. 7563 * In that case it is permitted to do the bulking 7564 * of multiple FIB entries to a single register write. 7565 */ 7566 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7567 fib_event->family == next_fib_event->family && 7568 fib_event->event == next_fib_event->event; 7569 op_ctx->event = fib_event->event; 7570 7571 /* In case family of this and the previous entry are different, context 7572 * reinitialization is going to be needed now, indicate that. 7573 * Note that since last_family is initialized to AF_UNSPEC, this is always 7574 * going to happen for the first entry processed in the work. 7575 */ 7576 if (fib_event->family != last_family) 7577 op_ctx->initialized = false; 7578 7579 switch (fib_event->family) { 7580 case AF_INET: 7581 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7582 fib_event); 7583 break; 7584 case AF_INET6: 7585 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7586 fib_event); 7587 break; 7588 case RTNL_FAMILY_IP6MR: 7589 case RTNL_FAMILY_IPMR: 7590 /* Unlock here as inside FIBMR the lock is taken again 7591 * under RTNL. The per-instance operation context 7592 * is not used by FIBMR. 7593 */ 7594 mutex_unlock(&router->lock); 7595 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7596 fib_event); 7597 mutex_lock(&router->lock); 7598 break; 7599 default: 7600 WARN_ON_ONCE(1); 7601 } 7602 last_family = fib_event->family; 7603 kfree(fib_event); 7604 cond_resched(); 7605 } 7606 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7607 mutex_unlock(&router->lock); 7608 } 7609 7610 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7611 struct fib_notifier_info *info) 7612 { 7613 struct fib_entry_notifier_info *fen_info; 7614 struct fib_nh_notifier_info *fnh_info; 7615 7616 switch (fib_event->event) { 7617 case FIB_EVENT_ENTRY_REPLACE: 7618 case FIB_EVENT_ENTRY_DEL: 7619 fen_info = container_of(info, struct fib_entry_notifier_info, 7620 info); 7621 fib_event->fen_info = *fen_info; 7622 /* Take reference on fib_info to prevent it from being 7623 * freed while event is queued. Release it afterwards. 7624 */ 7625 fib_info_hold(fib_event->fen_info.fi); 7626 break; 7627 case FIB_EVENT_NH_ADD: 7628 case FIB_EVENT_NH_DEL: 7629 fnh_info = container_of(info, struct fib_nh_notifier_info, 7630 info); 7631 fib_event->fnh_info = *fnh_info; 7632 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7633 break; 7634 } 7635 } 7636 7637 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7638 struct fib_notifier_info *info) 7639 { 7640 struct fib6_entry_notifier_info *fen6_info; 7641 int err; 7642 7643 switch (fib_event->event) { 7644 case FIB_EVENT_ENTRY_REPLACE: 7645 case FIB_EVENT_ENTRY_APPEND: 7646 case FIB_EVENT_ENTRY_DEL: 7647 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7648 info); 7649 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7650 fen6_info); 7651 if (err) 7652 return err; 7653 break; 7654 } 7655 7656 return 0; 7657 } 7658 7659 static void 7660 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7661 struct fib_notifier_info *info) 7662 { 7663 switch (fib_event->event) { 7664 case FIB_EVENT_ENTRY_REPLACE: 7665 case FIB_EVENT_ENTRY_ADD: 7666 case FIB_EVENT_ENTRY_DEL: 7667 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7668 mr_cache_hold(fib_event->men_info.mfc); 7669 break; 7670 case FIB_EVENT_VIF_ADD: 7671 case FIB_EVENT_VIF_DEL: 7672 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7673 dev_hold(fib_event->ven_info.dev); 7674 break; 7675 } 7676 } 7677 7678 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7679 struct fib_notifier_info *info, 7680 struct mlxsw_sp *mlxsw_sp) 7681 { 7682 struct netlink_ext_ack *extack = info->extack; 7683 struct fib_rule_notifier_info *fr_info; 7684 struct fib_rule *rule; 7685 int err = 0; 7686 7687 /* nothing to do at the moment */ 7688 if (event == FIB_EVENT_RULE_DEL) 7689 return 0; 7690 7691 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7692 rule = fr_info->rule; 7693 7694 /* Rule only affects locally generated traffic */ 7695 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7696 return 0; 7697 7698 switch (info->family) { 7699 case AF_INET: 7700 if (!fib4_rule_default(rule) && !rule->l3mdev) 7701 err = -EOPNOTSUPP; 7702 break; 7703 case AF_INET6: 7704 if (!fib6_rule_default(rule) && !rule->l3mdev) 7705 err = -EOPNOTSUPP; 7706 break; 7707 case RTNL_FAMILY_IPMR: 7708 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7709 err = -EOPNOTSUPP; 7710 break; 7711 case RTNL_FAMILY_IP6MR: 7712 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7713 err = -EOPNOTSUPP; 7714 break; 7715 } 7716 7717 if (err < 0) 7718 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7719 7720 return err; 7721 } 7722 7723 /* Called with rcu_read_lock() */ 7724 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7725 unsigned long event, void *ptr) 7726 { 7727 struct mlxsw_sp_fib_event *fib_event; 7728 struct fib_notifier_info *info = ptr; 7729 struct mlxsw_sp_router *router; 7730 int err; 7731 7732 if ((info->family != AF_INET && info->family != AF_INET6 && 7733 info->family != RTNL_FAMILY_IPMR && 7734 info->family != RTNL_FAMILY_IP6MR)) 7735 return NOTIFY_DONE; 7736 7737 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7738 7739 switch (event) { 7740 case FIB_EVENT_RULE_ADD: 7741 case FIB_EVENT_RULE_DEL: 7742 err = mlxsw_sp_router_fib_rule_event(event, info, 7743 router->mlxsw_sp); 7744 return notifier_from_errno(err); 7745 case FIB_EVENT_ENTRY_ADD: 7746 case FIB_EVENT_ENTRY_REPLACE: 7747 case FIB_EVENT_ENTRY_APPEND: 7748 if (info->family == AF_INET) { 7749 struct fib_entry_notifier_info *fen_info = ptr; 7750 7751 if (fen_info->fi->fib_nh_is_v6) { 7752 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7753 return notifier_from_errno(-EINVAL); 7754 } 7755 } 7756 break; 7757 } 7758 7759 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7760 if (!fib_event) 7761 return NOTIFY_BAD; 7762 7763 fib_event->mlxsw_sp = router->mlxsw_sp; 7764 fib_event->event = event; 7765 fib_event->family = info->family; 7766 7767 switch (info->family) { 7768 case AF_INET: 7769 mlxsw_sp_router_fib4_event(fib_event, info); 7770 break; 7771 case AF_INET6: 7772 err = mlxsw_sp_router_fib6_event(fib_event, info); 7773 if (err) 7774 goto err_fib_event; 7775 break; 7776 case RTNL_FAMILY_IP6MR: 7777 case RTNL_FAMILY_IPMR: 7778 mlxsw_sp_router_fibmr_event(fib_event, info); 7779 break; 7780 } 7781 7782 /* Enqueue the event and trigger the work */ 7783 spin_lock_bh(&router->fib_event_queue_lock); 7784 list_add_tail(&fib_event->list, &router->fib_event_queue); 7785 spin_unlock_bh(&router->fib_event_queue_lock); 7786 mlxsw_core_schedule_work(&router->fib_event_work); 7787 7788 return NOTIFY_DONE; 7789 7790 err_fib_event: 7791 kfree(fib_event); 7792 return NOTIFY_BAD; 7793 } 7794 7795 static struct mlxsw_sp_rif * 7796 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7797 const struct net_device *dev) 7798 { 7799 int i; 7800 7801 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7802 if (mlxsw_sp->router->rifs[i] && 7803 mlxsw_sp->router->rifs[i]->dev == dev) 7804 return mlxsw_sp->router->rifs[i]; 7805 7806 return NULL; 7807 } 7808 7809 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7810 const struct net_device *dev) 7811 { 7812 struct mlxsw_sp_rif *rif; 7813 7814 mutex_lock(&mlxsw_sp->router->lock); 7815 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7816 mutex_unlock(&mlxsw_sp->router->lock); 7817 7818 return rif; 7819 } 7820 7821 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7822 { 7823 struct mlxsw_sp_rif *rif; 7824 u16 vid = 0; 7825 7826 mutex_lock(&mlxsw_sp->router->lock); 7827 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7828 if (!rif) 7829 goto out; 7830 7831 /* We only return the VID for VLAN RIFs. Otherwise we return an 7832 * invalid value (0). 7833 */ 7834 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7835 goto out; 7836 7837 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7838 7839 out: 7840 mutex_unlock(&mlxsw_sp->router->lock); 7841 return vid; 7842 } 7843 7844 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7845 { 7846 char ritr_pl[MLXSW_REG_RITR_LEN]; 7847 int err; 7848 7849 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7850 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7851 if (err) 7852 return err; 7853 7854 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7855 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7856 } 7857 7858 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7859 struct mlxsw_sp_rif *rif) 7860 { 7861 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7862 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7863 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7864 } 7865 7866 static bool 7867 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7868 unsigned long event) 7869 { 7870 struct inet6_dev *inet6_dev; 7871 bool addr_list_empty = true; 7872 struct in_device *idev; 7873 7874 switch (event) { 7875 case NETDEV_UP: 7876 return rif == NULL; 7877 case NETDEV_DOWN: 7878 rcu_read_lock(); 7879 idev = __in_dev_get_rcu(dev); 7880 if (idev && idev->ifa_list) 7881 addr_list_empty = false; 7882 7883 inet6_dev = __in6_dev_get(dev); 7884 if (addr_list_empty && inet6_dev && 7885 !list_empty(&inet6_dev->addr_list)) 7886 addr_list_empty = false; 7887 rcu_read_unlock(); 7888 7889 /* macvlans do not have a RIF, but rather piggy back on the 7890 * RIF of their lower device. 7891 */ 7892 if (netif_is_macvlan(dev) && addr_list_empty) 7893 return true; 7894 7895 if (rif && addr_list_empty && 7896 !netif_is_l3_slave(rif->dev)) 7897 return true; 7898 /* It is possible we already removed the RIF ourselves 7899 * if it was assigned to a netdev that is now a bridge 7900 * or LAG slave. 7901 */ 7902 return false; 7903 } 7904 7905 return false; 7906 } 7907 7908 static enum mlxsw_sp_rif_type 7909 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7910 const struct net_device *dev) 7911 { 7912 enum mlxsw_sp_fid_type type; 7913 7914 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7915 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7916 7917 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7918 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7919 type = MLXSW_SP_FID_TYPE_8021Q; 7920 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7921 type = MLXSW_SP_FID_TYPE_8021Q; 7922 else if (netif_is_bridge_master(dev)) 7923 type = MLXSW_SP_FID_TYPE_8021D; 7924 else 7925 type = MLXSW_SP_FID_TYPE_RFID; 7926 7927 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7928 } 7929 7930 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 7931 { 7932 int i; 7933 7934 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 7935 if (!mlxsw_sp->router->rifs[i]) { 7936 *p_rif_index = i; 7937 return 0; 7938 } 7939 } 7940 7941 return -ENOBUFS; 7942 } 7943 7944 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 7945 u16 vr_id, 7946 struct net_device *l3_dev) 7947 { 7948 struct mlxsw_sp_rif *rif; 7949 7950 rif = kzalloc(rif_size, GFP_KERNEL); 7951 if (!rif) 7952 return NULL; 7953 7954 INIT_LIST_HEAD(&rif->nexthop_list); 7955 INIT_LIST_HEAD(&rif->neigh_list); 7956 if (l3_dev) { 7957 ether_addr_copy(rif->addr, l3_dev->dev_addr); 7958 rif->mtu = l3_dev->mtu; 7959 rif->dev = l3_dev; 7960 } 7961 rif->vr_id = vr_id; 7962 rif->rif_index = rif_index; 7963 7964 return rif; 7965 } 7966 7967 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 7968 u16 rif_index) 7969 { 7970 return mlxsw_sp->router->rifs[rif_index]; 7971 } 7972 7973 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 7974 { 7975 return rif->rif_index; 7976 } 7977 7978 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7979 { 7980 return lb_rif->common.rif_index; 7981 } 7982 7983 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7984 { 7985 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 7986 struct mlxsw_sp_vr *ul_vr; 7987 7988 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 7989 if (WARN_ON(IS_ERR(ul_vr))) 7990 return 0; 7991 7992 return ul_vr->id; 7993 } 7994 7995 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7996 { 7997 return lb_rif->ul_rif_id; 7998 } 7999 8000 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8001 { 8002 return rif->dev->ifindex; 8003 } 8004 8005 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 8006 { 8007 return rif->dev; 8008 } 8009 8010 static struct mlxsw_sp_rif * 8011 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8012 const struct mlxsw_sp_rif_params *params, 8013 struct netlink_ext_ack *extack) 8014 { 8015 u32 tb_id = l3mdev_fib_table(params->dev); 8016 const struct mlxsw_sp_rif_ops *ops; 8017 struct mlxsw_sp_fid *fid = NULL; 8018 enum mlxsw_sp_rif_type type; 8019 struct mlxsw_sp_rif *rif; 8020 struct mlxsw_sp_vr *vr; 8021 u16 rif_index; 8022 int i, err; 8023 8024 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8025 ops = mlxsw_sp->router->rif_ops_arr[type]; 8026 8027 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8028 if (IS_ERR(vr)) 8029 return ERR_CAST(vr); 8030 vr->rif_count++; 8031 8032 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8033 if (err) { 8034 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8035 goto err_rif_index_alloc; 8036 } 8037 8038 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 8039 if (!rif) { 8040 err = -ENOMEM; 8041 goto err_rif_alloc; 8042 } 8043 dev_hold(rif->dev); 8044 mlxsw_sp->router->rifs[rif_index] = rif; 8045 rif->mlxsw_sp = mlxsw_sp; 8046 rif->ops = ops; 8047 8048 if (ops->fid_get) { 8049 fid = ops->fid_get(rif, extack); 8050 if (IS_ERR(fid)) { 8051 err = PTR_ERR(fid); 8052 goto err_fid_get; 8053 } 8054 rif->fid = fid; 8055 } 8056 8057 if (ops->setup) 8058 ops->setup(rif, params); 8059 8060 err = ops->configure(rif); 8061 if (err) 8062 goto err_configure; 8063 8064 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8065 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8066 if (err) 8067 goto err_mr_rif_add; 8068 } 8069 8070 mlxsw_sp_rif_counters_alloc(rif); 8071 8072 return rif; 8073 8074 err_mr_rif_add: 8075 for (i--; i >= 0; i--) 8076 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8077 ops->deconfigure(rif); 8078 err_configure: 8079 if (fid) 8080 mlxsw_sp_fid_put(fid); 8081 err_fid_get: 8082 mlxsw_sp->router->rifs[rif_index] = NULL; 8083 dev_put(rif->dev); 8084 kfree(rif); 8085 err_rif_alloc: 8086 err_rif_index_alloc: 8087 vr->rif_count--; 8088 mlxsw_sp_vr_put(mlxsw_sp, vr); 8089 return ERR_PTR(err); 8090 } 8091 8092 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8093 { 8094 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8095 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8096 struct mlxsw_sp_fid *fid = rif->fid; 8097 struct mlxsw_sp_vr *vr; 8098 int i; 8099 8100 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8101 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8102 8103 mlxsw_sp_rif_counters_free(rif); 8104 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8105 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8106 ops->deconfigure(rif); 8107 if (fid) 8108 /* Loopback RIFs are not associated with a FID. */ 8109 mlxsw_sp_fid_put(fid); 8110 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8111 dev_put(rif->dev); 8112 kfree(rif); 8113 vr->rif_count--; 8114 mlxsw_sp_vr_put(mlxsw_sp, vr); 8115 } 8116 8117 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8118 struct net_device *dev) 8119 { 8120 struct mlxsw_sp_rif *rif; 8121 8122 mutex_lock(&mlxsw_sp->router->lock); 8123 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8124 if (!rif) 8125 goto out; 8126 mlxsw_sp_rif_destroy(rif); 8127 out: 8128 mutex_unlock(&mlxsw_sp->router->lock); 8129 } 8130 8131 static void 8132 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8133 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8134 { 8135 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8136 8137 params->vid = mlxsw_sp_port_vlan->vid; 8138 params->lag = mlxsw_sp_port->lagged; 8139 if (params->lag) 8140 params->lag_id = mlxsw_sp_port->lag_id; 8141 else 8142 params->system_port = mlxsw_sp_port->local_port; 8143 } 8144 8145 static struct mlxsw_sp_rif_subport * 8146 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8147 { 8148 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8149 } 8150 8151 static struct mlxsw_sp_rif * 8152 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8153 const struct mlxsw_sp_rif_params *params, 8154 struct netlink_ext_ack *extack) 8155 { 8156 struct mlxsw_sp_rif_subport *rif_subport; 8157 struct mlxsw_sp_rif *rif; 8158 8159 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8160 if (!rif) 8161 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8162 8163 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8164 refcount_inc(&rif_subport->ref_count); 8165 return rif; 8166 } 8167 8168 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8169 { 8170 struct mlxsw_sp_rif_subport *rif_subport; 8171 8172 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8173 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8174 return; 8175 8176 mlxsw_sp_rif_destroy(rif); 8177 } 8178 8179 static int 8180 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8181 struct net_device *l3_dev, 8182 struct netlink_ext_ack *extack) 8183 { 8184 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8185 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8186 struct mlxsw_sp_rif_params params = { 8187 .dev = l3_dev, 8188 }; 8189 u16 vid = mlxsw_sp_port_vlan->vid; 8190 struct mlxsw_sp_rif *rif; 8191 struct mlxsw_sp_fid *fid; 8192 int err; 8193 8194 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8195 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8196 if (IS_ERR(rif)) 8197 return PTR_ERR(rif); 8198 8199 /* FID was already created, just take a reference */ 8200 fid = rif->ops->fid_get(rif, extack); 8201 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8202 if (err) 8203 goto err_fid_port_vid_map; 8204 8205 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8206 if (err) 8207 goto err_port_vid_learning_set; 8208 8209 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8210 BR_STATE_FORWARDING); 8211 if (err) 8212 goto err_port_vid_stp_set; 8213 8214 mlxsw_sp_port_vlan->fid = fid; 8215 8216 return 0; 8217 8218 err_port_vid_stp_set: 8219 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8220 err_port_vid_learning_set: 8221 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8222 err_fid_port_vid_map: 8223 mlxsw_sp_fid_put(fid); 8224 mlxsw_sp_rif_subport_put(rif); 8225 return err; 8226 } 8227 8228 static void 8229 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8230 { 8231 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8232 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8233 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8234 u16 vid = mlxsw_sp_port_vlan->vid; 8235 8236 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8237 return; 8238 8239 mlxsw_sp_port_vlan->fid = NULL; 8240 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8241 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8242 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8243 mlxsw_sp_fid_put(fid); 8244 mlxsw_sp_rif_subport_put(rif); 8245 } 8246 8247 int 8248 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8249 struct net_device *l3_dev, 8250 struct netlink_ext_ack *extack) 8251 { 8252 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8253 struct mlxsw_sp_rif *rif; 8254 int err = 0; 8255 8256 mutex_lock(&mlxsw_sp->router->lock); 8257 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8258 if (!rif) 8259 goto out; 8260 8261 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8262 extack); 8263 out: 8264 mutex_unlock(&mlxsw_sp->router->lock); 8265 return err; 8266 } 8267 8268 void 8269 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8270 { 8271 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8272 8273 mutex_lock(&mlxsw_sp->router->lock); 8274 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8275 mutex_unlock(&mlxsw_sp->router->lock); 8276 } 8277 8278 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8279 struct net_device *port_dev, 8280 unsigned long event, u16 vid, 8281 struct netlink_ext_ack *extack) 8282 { 8283 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8284 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8285 8286 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8287 if (WARN_ON(!mlxsw_sp_port_vlan)) 8288 return -EINVAL; 8289 8290 switch (event) { 8291 case NETDEV_UP: 8292 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8293 l3_dev, extack); 8294 case NETDEV_DOWN: 8295 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8296 break; 8297 } 8298 8299 return 0; 8300 } 8301 8302 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8303 unsigned long event, 8304 struct netlink_ext_ack *extack) 8305 { 8306 if (netif_is_bridge_port(port_dev) || 8307 netif_is_lag_port(port_dev) || 8308 netif_is_ovs_port(port_dev)) 8309 return 0; 8310 8311 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8312 MLXSW_SP_DEFAULT_VID, extack); 8313 } 8314 8315 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8316 struct net_device *lag_dev, 8317 unsigned long event, u16 vid, 8318 struct netlink_ext_ack *extack) 8319 { 8320 struct net_device *port_dev; 8321 struct list_head *iter; 8322 int err; 8323 8324 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8325 if (mlxsw_sp_port_dev_check(port_dev)) { 8326 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8327 port_dev, 8328 event, vid, 8329 extack); 8330 if (err) 8331 return err; 8332 } 8333 } 8334 8335 return 0; 8336 } 8337 8338 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8339 unsigned long event, 8340 struct netlink_ext_ack *extack) 8341 { 8342 if (netif_is_bridge_port(lag_dev)) 8343 return 0; 8344 8345 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8346 MLXSW_SP_DEFAULT_VID, extack); 8347 } 8348 8349 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8350 struct net_device *l3_dev, 8351 unsigned long event, 8352 struct netlink_ext_ack *extack) 8353 { 8354 struct mlxsw_sp_rif_params params = { 8355 .dev = l3_dev, 8356 }; 8357 struct mlxsw_sp_rif *rif; 8358 8359 switch (event) { 8360 case NETDEV_UP: 8361 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8362 u16 proto; 8363 8364 br_vlan_get_proto(l3_dev, &proto); 8365 if (proto == ETH_P_8021AD) { 8366 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8367 return -EOPNOTSUPP; 8368 } 8369 } 8370 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8371 if (IS_ERR(rif)) 8372 return PTR_ERR(rif); 8373 break; 8374 case NETDEV_DOWN: 8375 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8376 mlxsw_sp_rif_destroy(rif); 8377 break; 8378 } 8379 8380 return 0; 8381 } 8382 8383 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8384 struct net_device *vlan_dev, 8385 unsigned long event, 8386 struct netlink_ext_ack *extack) 8387 { 8388 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8389 u16 vid = vlan_dev_vlan_id(vlan_dev); 8390 8391 if (netif_is_bridge_port(vlan_dev)) 8392 return 0; 8393 8394 if (mlxsw_sp_port_dev_check(real_dev)) 8395 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8396 event, vid, extack); 8397 else if (netif_is_lag_master(real_dev)) 8398 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8399 vid, extack); 8400 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8401 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8402 extack); 8403 8404 return 0; 8405 } 8406 8407 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8408 { 8409 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8410 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8411 8412 return ether_addr_equal_masked(mac, vrrp4, mask); 8413 } 8414 8415 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8416 { 8417 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8418 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8419 8420 return ether_addr_equal_masked(mac, vrrp6, mask); 8421 } 8422 8423 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8424 const u8 *mac, bool adding) 8425 { 8426 char ritr_pl[MLXSW_REG_RITR_LEN]; 8427 u8 vrrp_id = adding ? mac[5] : 0; 8428 int err; 8429 8430 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8431 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8432 return 0; 8433 8434 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8435 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8436 if (err) 8437 return err; 8438 8439 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8440 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8441 else 8442 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8443 8444 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8445 } 8446 8447 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8448 const struct net_device *macvlan_dev, 8449 struct netlink_ext_ack *extack) 8450 { 8451 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8452 struct mlxsw_sp_rif *rif; 8453 int err; 8454 8455 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8456 if (!rif) { 8457 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8458 return -EOPNOTSUPP; 8459 } 8460 8461 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8462 mlxsw_sp_fid_index(rif->fid), true); 8463 if (err) 8464 return err; 8465 8466 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8467 macvlan_dev->dev_addr, true); 8468 if (err) 8469 goto err_rif_vrrp_add; 8470 8471 /* Make sure the bridge driver does not have this MAC pointing at 8472 * some other port. 8473 */ 8474 if (rif->ops->fdb_del) 8475 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8476 8477 return 0; 8478 8479 err_rif_vrrp_add: 8480 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8481 mlxsw_sp_fid_index(rif->fid), false); 8482 return err; 8483 } 8484 8485 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8486 const struct net_device *macvlan_dev) 8487 { 8488 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8489 struct mlxsw_sp_rif *rif; 8490 8491 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8492 /* If we do not have a RIF, then we already took care of 8493 * removing the macvlan's MAC during RIF deletion. 8494 */ 8495 if (!rif) 8496 return; 8497 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8498 false); 8499 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8500 mlxsw_sp_fid_index(rif->fid), false); 8501 } 8502 8503 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8504 const struct net_device *macvlan_dev) 8505 { 8506 mutex_lock(&mlxsw_sp->router->lock); 8507 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8508 mutex_unlock(&mlxsw_sp->router->lock); 8509 } 8510 8511 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8512 struct net_device *macvlan_dev, 8513 unsigned long event, 8514 struct netlink_ext_ack *extack) 8515 { 8516 switch (event) { 8517 case NETDEV_UP: 8518 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8519 case NETDEV_DOWN: 8520 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8521 break; 8522 } 8523 8524 return 0; 8525 } 8526 8527 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 8528 struct net_device *dev, 8529 const unsigned char *dev_addr, 8530 struct netlink_ext_ack *extack) 8531 { 8532 struct mlxsw_sp_rif *rif; 8533 int i; 8534 8535 /* A RIF is not created for macvlan netdevs. Their MAC is used to 8536 * populate the FDB 8537 */ 8538 if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) 8539 return 0; 8540 8541 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8542 rif = mlxsw_sp->router->rifs[i]; 8543 if (rif && rif->ops && 8544 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB) 8545 continue; 8546 if (rif && rif->dev && rif->dev != dev && 8547 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 8548 mlxsw_sp->mac_mask)) { 8549 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 8550 return -EINVAL; 8551 } 8552 } 8553 8554 return 0; 8555 } 8556 8557 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8558 struct net_device *dev, 8559 unsigned long event, 8560 struct netlink_ext_ack *extack) 8561 { 8562 if (mlxsw_sp_port_dev_check(dev)) 8563 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8564 else if (netif_is_lag_master(dev)) 8565 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8566 else if (netif_is_bridge_master(dev)) 8567 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8568 extack); 8569 else if (is_vlan_dev(dev)) 8570 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8571 extack); 8572 else if (netif_is_macvlan(dev)) 8573 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8574 extack); 8575 else 8576 return 0; 8577 } 8578 8579 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8580 unsigned long event, void *ptr) 8581 { 8582 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8583 struct net_device *dev = ifa->ifa_dev->dev; 8584 struct mlxsw_sp_router *router; 8585 struct mlxsw_sp_rif *rif; 8586 int err = 0; 8587 8588 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8589 if (event == NETDEV_UP) 8590 return NOTIFY_DONE; 8591 8592 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8593 mutex_lock(&router->lock); 8594 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8595 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8596 goto out; 8597 8598 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8599 out: 8600 mutex_unlock(&router->lock); 8601 return notifier_from_errno(err); 8602 } 8603 8604 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8605 unsigned long event, void *ptr) 8606 { 8607 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8608 struct net_device *dev = ivi->ivi_dev->dev; 8609 struct mlxsw_sp *mlxsw_sp; 8610 struct mlxsw_sp_rif *rif; 8611 int err = 0; 8612 8613 mlxsw_sp = mlxsw_sp_lower_get(dev); 8614 if (!mlxsw_sp) 8615 return NOTIFY_DONE; 8616 8617 mutex_lock(&mlxsw_sp->router->lock); 8618 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8619 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8620 goto out; 8621 8622 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8623 ivi->extack); 8624 if (err) 8625 goto out; 8626 8627 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8628 out: 8629 mutex_unlock(&mlxsw_sp->router->lock); 8630 return notifier_from_errno(err); 8631 } 8632 8633 struct mlxsw_sp_inet6addr_event_work { 8634 struct work_struct work; 8635 struct mlxsw_sp *mlxsw_sp; 8636 struct net_device *dev; 8637 unsigned long event; 8638 }; 8639 8640 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8641 { 8642 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8643 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8644 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8645 struct net_device *dev = inet6addr_work->dev; 8646 unsigned long event = inet6addr_work->event; 8647 struct mlxsw_sp_rif *rif; 8648 8649 rtnl_lock(); 8650 mutex_lock(&mlxsw_sp->router->lock); 8651 8652 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8653 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8654 goto out; 8655 8656 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8657 out: 8658 mutex_unlock(&mlxsw_sp->router->lock); 8659 rtnl_unlock(); 8660 dev_put(dev); 8661 kfree(inet6addr_work); 8662 } 8663 8664 /* Called with rcu_read_lock() */ 8665 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8666 unsigned long event, void *ptr) 8667 { 8668 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8669 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8670 struct net_device *dev = if6->idev->dev; 8671 struct mlxsw_sp_router *router; 8672 8673 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8674 if (event == NETDEV_UP) 8675 return NOTIFY_DONE; 8676 8677 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8678 if (!inet6addr_work) 8679 return NOTIFY_BAD; 8680 8681 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8682 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8683 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8684 inet6addr_work->dev = dev; 8685 inet6addr_work->event = event; 8686 dev_hold(dev); 8687 mlxsw_core_schedule_work(&inet6addr_work->work); 8688 8689 return NOTIFY_DONE; 8690 } 8691 8692 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8693 unsigned long event, void *ptr) 8694 { 8695 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8696 struct net_device *dev = i6vi->i6vi_dev->dev; 8697 struct mlxsw_sp *mlxsw_sp; 8698 struct mlxsw_sp_rif *rif; 8699 int err = 0; 8700 8701 mlxsw_sp = mlxsw_sp_lower_get(dev); 8702 if (!mlxsw_sp) 8703 return NOTIFY_DONE; 8704 8705 mutex_lock(&mlxsw_sp->router->lock); 8706 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8707 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8708 goto out; 8709 8710 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8711 i6vi->extack); 8712 if (err) 8713 goto out; 8714 8715 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8716 out: 8717 mutex_unlock(&mlxsw_sp->router->lock); 8718 return notifier_from_errno(err); 8719 } 8720 8721 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8722 const char *mac, int mtu) 8723 { 8724 char ritr_pl[MLXSW_REG_RITR_LEN]; 8725 int err; 8726 8727 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8728 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8729 if (err) 8730 return err; 8731 8732 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8733 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8734 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8735 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8736 } 8737 8738 static int 8739 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8740 struct mlxsw_sp_rif *rif) 8741 { 8742 struct net_device *dev = rif->dev; 8743 u16 fid_index; 8744 int err; 8745 8746 fid_index = mlxsw_sp_fid_index(rif->fid); 8747 8748 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8749 if (err) 8750 return err; 8751 8752 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8753 dev->mtu); 8754 if (err) 8755 goto err_rif_edit; 8756 8757 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 8758 if (err) 8759 goto err_rif_fdb_op; 8760 8761 if (rif->mtu != dev->mtu) { 8762 struct mlxsw_sp_vr *vr; 8763 int i; 8764 8765 /* The RIF is relevant only to its mr_table instance, as unlike 8766 * unicast routing, in multicast routing a RIF cannot be shared 8767 * between several multicast routing tables. 8768 */ 8769 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8770 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8771 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 8772 rif, dev->mtu); 8773 } 8774 8775 ether_addr_copy(rif->addr, dev->dev_addr); 8776 rif->mtu = dev->mtu; 8777 8778 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 8779 8780 return 0; 8781 8782 err_rif_fdb_op: 8783 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 8784 err_rif_edit: 8785 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 8786 return err; 8787 } 8788 8789 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 8790 struct netdev_notifier_pre_changeaddr_info *info) 8791 { 8792 struct netlink_ext_ack *extack; 8793 8794 extack = netdev_notifier_info_to_extack(&info->info); 8795 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 8796 info->dev_addr, extack); 8797 } 8798 8799 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 8800 unsigned long event, void *ptr) 8801 { 8802 struct mlxsw_sp *mlxsw_sp; 8803 struct mlxsw_sp_rif *rif; 8804 int err = 0; 8805 8806 mlxsw_sp = mlxsw_sp_lower_get(dev); 8807 if (!mlxsw_sp) 8808 return 0; 8809 8810 mutex_lock(&mlxsw_sp->router->lock); 8811 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8812 if (!rif) 8813 goto out; 8814 8815 switch (event) { 8816 case NETDEV_CHANGEMTU: 8817 case NETDEV_CHANGEADDR: 8818 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 8819 break; 8820 case NETDEV_PRE_CHANGEADDR: 8821 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 8822 break; 8823 } 8824 8825 out: 8826 mutex_unlock(&mlxsw_sp->router->lock); 8827 return err; 8828 } 8829 8830 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 8831 struct net_device *l3_dev, 8832 struct netlink_ext_ack *extack) 8833 { 8834 struct mlxsw_sp_rif *rif; 8835 8836 /* If netdev is already associated with a RIF, then we need to 8837 * destroy it and create a new one with the new virtual router ID. 8838 */ 8839 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8840 if (rif) 8841 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 8842 extack); 8843 8844 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 8845 } 8846 8847 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 8848 struct net_device *l3_dev) 8849 { 8850 struct mlxsw_sp_rif *rif; 8851 8852 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8853 if (!rif) 8854 return; 8855 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 8856 } 8857 8858 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 8859 struct netdev_notifier_changeupper_info *info) 8860 { 8861 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 8862 int err = 0; 8863 8864 /* We do not create a RIF for a macvlan, but only use it to 8865 * direct more MAC addresses to the router. 8866 */ 8867 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 8868 return 0; 8869 8870 mutex_lock(&mlxsw_sp->router->lock); 8871 switch (event) { 8872 case NETDEV_PRECHANGEUPPER: 8873 break; 8874 case NETDEV_CHANGEUPPER: 8875 if (info->linking) { 8876 struct netlink_ext_ack *extack; 8877 8878 extack = netdev_notifier_info_to_extack(&info->info); 8879 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 8880 } else { 8881 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 8882 } 8883 break; 8884 } 8885 mutex_unlock(&mlxsw_sp->router->lock); 8886 8887 return err; 8888 } 8889 8890 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 8891 struct netdev_nested_priv *priv) 8892 { 8893 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 8894 8895 if (!netif_is_macvlan(dev)) 8896 return 0; 8897 8898 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 8899 mlxsw_sp_fid_index(rif->fid), false); 8900 } 8901 8902 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 8903 { 8904 struct netdev_nested_priv priv = { 8905 .data = (void *)rif, 8906 }; 8907 8908 if (!netif_is_macvlan_port(rif->dev)) 8909 return 0; 8910 8911 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 8912 return netdev_walk_all_upper_dev_rcu(rif->dev, 8913 __mlxsw_sp_rif_macvlan_flush, &priv); 8914 } 8915 8916 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 8917 const struct mlxsw_sp_rif_params *params) 8918 { 8919 struct mlxsw_sp_rif_subport *rif_subport; 8920 8921 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8922 refcount_set(&rif_subport->ref_count, 1); 8923 rif_subport->vid = params->vid; 8924 rif_subport->lag = params->lag; 8925 if (params->lag) 8926 rif_subport->lag_id = params->lag_id; 8927 else 8928 rif_subport->system_port = params->system_port; 8929 } 8930 8931 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 8932 { 8933 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8934 struct mlxsw_sp_rif_subport *rif_subport; 8935 char ritr_pl[MLXSW_REG_RITR_LEN]; 8936 8937 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8938 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 8939 rif->rif_index, rif->vr_id, rif->dev->mtu); 8940 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 8941 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 8942 rif_subport->lag ? rif_subport->lag_id : 8943 rif_subport->system_port, 8944 rif_subport->vid); 8945 8946 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8947 } 8948 8949 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 8950 { 8951 int err; 8952 8953 err = mlxsw_sp_rif_subport_op(rif, true); 8954 if (err) 8955 return err; 8956 8957 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8958 mlxsw_sp_fid_index(rif->fid), true); 8959 if (err) 8960 goto err_rif_fdb_op; 8961 8962 mlxsw_sp_fid_rif_set(rif->fid, rif); 8963 return 0; 8964 8965 err_rif_fdb_op: 8966 mlxsw_sp_rif_subport_op(rif, false); 8967 return err; 8968 } 8969 8970 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 8971 { 8972 struct mlxsw_sp_fid *fid = rif->fid; 8973 8974 mlxsw_sp_fid_rif_set(fid, NULL); 8975 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 8976 mlxsw_sp_fid_index(fid), false); 8977 mlxsw_sp_rif_macvlan_flush(rif); 8978 mlxsw_sp_rif_subport_op(rif, false); 8979 } 8980 8981 static struct mlxsw_sp_fid * 8982 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 8983 struct netlink_ext_ack *extack) 8984 { 8985 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 8986 } 8987 8988 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 8989 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 8990 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 8991 .setup = mlxsw_sp_rif_subport_setup, 8992 .configure = mlxsw_sp_rif_subport_configure, 8993 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 8994 .fid_get = mlxsw_sp_rif_subport_fid_get, 8995 }; 8996 8997 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 8998 enum mlxsw_reg_ritr_if_type type, 8999 u16 vid_fid, bool enable) 9000 { 9001 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9002 char ritr_pl[MLXSW_REG_RITR_LEN]; 9003 9004 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9005 rif->dev->mtu); 9006 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9007 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 9008 9009 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9010 } 9011 9012 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9013 { 9014 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9015 } 9016 9017 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 9018 { 9019 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9020 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9021 int err; 9022 9023 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 9024 true); 9025 if (err) 9026 return err; 9027 9028 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9029 mlxsw_sp_router_port(mlxsw_sp), true); 9030 if (err) 9031 goto err_fid_mc_flood_set; 9032 9033 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9034 mlxsw_sp_router_port(mlxsw_sp), true); 9035 if (err) 9036 goto err_fid_bc_flood_set; 9037 9038 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9039 mlxsw_sp_fid_index(rif->fid), true); 9040 if (err) 9041 goto err_rif_fdb_op; 9042 9043 mlxsw_sp_fid_rif_set(rif->fid, rif); 9044 return 0; 9045 9046 err_rif_fdb_op: 9047 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9048 mlxsw_sp_router_port(mlxsw_sp), false); 9049 err_fid_bc_flood_set: 9050 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9051 mlxsw_sp_router_port(mlxsw_sp), false); 9052 err_fid_mc_flood_set: 9053 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9054 return err; 9055 } 9056 9057 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9058 { 9059 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9060 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9061 struct mlxsw_sp_fid *fid = rif->fid; 9062 9063 mlxsw_sp_fid_rif_set(fid, NULL); 9064 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9065 mlxsw_sp_fid_index(fid), false); 9066 mlxsw_sp_rif_macvlan_flush(rif); 9067 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9068 mlxsw_sp_router_port(mlxsw_sp), false); 9069 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9070 mlxsw_sp_router_port(mlxsw_sp), false); 9071 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9072 } 9073 9074 static struct mlxsw_sp_fid * 9075 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9076 struct netlink_ext_ack *extack) 9077 { 9078 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 9079 } 9080 9081 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9082 { 9083 struct switchdev_notifier_fdb_info info; 9084 struct net_device *dev; 9085 9086 dev = br_fdb_find_port(rif->dev, mac, 0); 9087 if (!dev) 9088 return; 9089 9090 info.addr = mac; 9091 info.vid = 0; 9092 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9093 NULL); 9094 } 9095 9096 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9097 .type = MLXSW_SP_RIF_TYPE_FID, 9098 .rif_size = sizeof(struct mlxsw_sp_rif), 9099 .configure = mlxsw_sp_rif_fid_configure, 9100 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9101 .fid_get = mlxsw_sp_rif_fid_fid_get, 9102 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9103 }; 9104 9105 static struct mlxsw_sp_fid * 9106 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9107 struct netlink_ext_ack *extack) 9108 { 9109 struct net_device *br_dev; 9110 u16 vid; 9111 int err; 9112 9113 if (is_vlan_dev(rif->dev)) { 9114 vid = vlan_dev_vlan_id(rif->dev); 9115 br_dev = vlan_dev_real_dev(rif->dev); 9116 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9117 return ERR_PTR(-EINVAL); 9118 } else { 9119 err = br_vlan_get_pvid(rif->dev, &vid); 9120 if (err < 0 || !vid) { 9121 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9122 return ERR_PTR(-EINVAL); 9123 } 9124 } 9125 9126 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9127 } 9128 9129 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9130 { 9131 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9132 struct switchdev_notifier_fdb_info info; 9133 struct net_device *br_dev; 9134 struct net_device *dev; 9135 9136 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 9137 dev = br_fdb_find_port(br_dev, mac, vid); 9138 if (!dev) 9139 return; 9140 9141 info.addr = mac; 9142 info.vid = vid; 9143 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9144 NULL); 9145 } 9146 9147 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 9148 .type = MLXSW_SP_RIF_TYPE_VLAN, 9149 .rif_size = sizeof(struct mlxsw_sp_rif), 9150 .configure = mlxsw_sp_rif_fid_configure, 9151 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9152 .fid_get = mlxsw_sp_rif_vlan_fid_get, 9153 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 9154 }; 9155 9156 static struct mlxsw_sp_rif_ipip_lb * 9157 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 9158 { 9159 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 9160 } 9161 9162 static void 9163 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 9164 const struct mlxsw_sp_rif_params *params) 9165 { 9166 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 9167 struct mlxsw_sp_rif_ipip_lb *rif_lb; 9168 9169 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 9170 common); 9171 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 9172 rif_lb->lb_config = params_lb->lb_config; 9173 } 9174 9175 static int 9176 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9177 { 9178 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9179 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9180 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9181 struct mlxsw_sp_vr *ul_vr; 9182 int err; 9183 9184 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 9185 if (IS_ERR(ul_vr)) 9186 return PTR_ERR(ul_vr); 9187 9188 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 9189 if (err) 9190 goto err_loopback_op; 9191 9192 lb_rif->ul_vr_id = ul_vr->id; 9193 lb_rif->ul_rif_id = 0; 9194 ++ul_vr->rif_count; 9195 return 0; 9196 9197 err_loopback_op: 9198 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9199 return err; 9200 } 9201 9202 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9203 { 9204 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9205 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9206 struct mlxsw_sp_vr *ul_vr; 9207 9208 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 9209 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 9210 9211 --ul_vr->rif_count; 9212 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9213 } 9214 9215 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 9216 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9217 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9218 .setup = mlxsw_sp_rif_ipip_lb_setup, 9219 .configure = mlxsw_sp1_rif_ipip_lb_configure, 9220 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 9221 }; 9222 9223 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 9224 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9225 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9226 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9227 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 9228 }; 9229 9230 static int 9231 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 9232 { 9233 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9234 char ritr_pl[MLXSW_REG_RITR_LEN]; 9235 9236 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 9237 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 9238 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 9239 MLXSW_REG_RITR_LOOPBACK_GENERIC); 9240 9241 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9242 } 9243 9244 static struct mlxsw_sp_rif * 9245 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 9246 struct netlink_ext_ack *extack) 9247 { 9248 struct mlxsw_sp_rif *ul_rif; 9249 u16 rif_index; 9250 int err; 9251 9252 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 9253 if (err) { 9254 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 9255 return ERR_PTR(err); 9256 } 9257 9258 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 9259 if (!ul_rif) 9260 return ERR_PTR(-ENOMEM); 9261 9262 mlxsw_sp->router->rifs[rif_index] = ul_rif; 9263 ul_rif->mlxsw_sp = mlxsw_sp; 9264 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 9265 if (err) 9266 goto ul_rif_op_err; 9267 9268 return ul_rif; 9269 9270 ul_rif_op_err: 9271 mlxsw_sp->router->rifs[rif_index] = NULL; 9272 kfree(ul_rif); 9273 return ERR_PTR(err); 9274 } 9275 9276 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9277 { 9278 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9279 9280 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9281 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9282 kfree(ul_rif); 9283 } 9284 9285 static struct mlxsw_sp_rif * 9286 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 9287 struct netlink_ext_ack *extack) 9288 { 9289 struct mlxsw_sp_vr *vr; 9290 int err; 9291 9292 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 9293 if (IS_ERR(vr)) 9294 return ERR_CAST(vr); 9295 9296 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 9297 return vr->ul_rif; 9298 9299 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 9300 if (IS_ERR(vr->ul_rif)) { 9301 err = PTR_ERR(vr->ul_rif); 9302 goto err_ul_rif_create; 9303 } 9304 9305 vr->rif_count++; 9306 refcount_set(&vr->ul_rif_refcnt, 1); 9307 9308 return vr->ul_rif; 9309 9310 err_ul_rif_create: 9311 mlxsw_sp_vr_put(mlxsw_sp, vr); 9312 return ERR_PTR(err); 9313 } 9314 9315 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 9316 { 9317 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9318 struct mlxsw_sp_vr *vr; 9319 9320 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 9321 9322 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 9323 return; 9324 9325 vr->rif_count--; 9326 mlxsw_sp_ul_rif_destroy(ul_rif); 9327 mlxsw_sp_vr_put(mlxsw_sp, vr); 9328 } 9329 9330 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 9331 u16 *ul_rif_index) 9332 { 9333 struct mlxsw_sp_rif *ul_rif; 9334 int err = 0; 9335 9336 mutex_lock(&mlxsw_sp->router->lock); 9337 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9338 if (IS_ERR(ul_rif)) { 9339 err = PTR_ERR(ul_rif); 9340 goto out; 9341 } 9342 *ul_rif_index = ul_rif->rif_index; 9343 out: 9344 mutex_unlock(&mlxsw_sp->router->lock); 9345 return err; 9346 } 9347 9348 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 9349 { 9350 struct mlxsw_sp_rif *ul_rif; 9351 9352 mutex_lock(&mlxsw_sp->router->lock); 9353 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 9354 if (WARN_ON(!ul_rif)) 9355 goto out; 9356 9357 mlxsw_sp_ul_rif_put(ul_rif); 9358 out: 9359 mutex_unlock(&mlxsw_sp->router->lock); 9360 } 9361 9362 static int 9363 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9364 { 9365 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9366 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9367 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9368 struct mlxsw_sp_rif *ul_rif; 9369 int err; 9370 9371 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9372 if (IS_ERR(ul_rif)) 9373 return PTR_ERR(ul_rif); 9374 9375 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 9376 if (err) 9377 goto err_loopback_op; 9378 9379 lb_rif->ul_vr_id = 0; 9380 lb_rif->ul_rif_id = ul_rif->rif_index; 9381 9382 return 0; 9383 9384 err_loopback_op: 9385 mlxsw_sp_ul_rif_put(ul_rif); 9386 return err; 9387 } 9388 9389 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9390 { 9391 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9392 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9393 struct mlxsw_sp_rif *ul_rif; 9394 9395 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 9396 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 9397 mlxsw_sp_ul_rif_put(ul_rif); 9398 } 9399 9400 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 9401 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9402 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9403 .setup = mlxsw_sp_rif_ipip_lb_setup, 9404 .configure = mlxsw_sp2_rif_ipip_lb_configure, 9405 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 9406 }; 9407 9408 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 9409 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9410 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9411 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9412 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 9413 }; 9414 9415 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 9416 { 9417 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9418 9419 mlxsw_sp->router->rifs = kcalloc(max_rifs, 9420 sizeof(struct mlxsw_sp_rif *), 9421 GFP_KERNEL); 9422 if (!mlxsw_sp->router->rifs) 9423 return -ENOMEM; 9424 9425 return 0; 9426 } 9427 9428 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 9429 { 9430 int i; 9431 9432 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 9433 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 9434 9435 kfree(mlxsw_sp->router->rifs); 9436 } 9437 9438 static int 9439 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 9440 { 9441 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 9442 9443 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 9444 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 9445 } 9446 9447 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 9448 { 9449 int err; 9450 9451 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 9452 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 9453 9454 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 9455 if (err) 9456 return err; 9457 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 9458 if (err) 9459 return err; 9460 9461 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9462 } 9463 9464 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9465 { 9466 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9467 } 9468 9469 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9470 { 9471 struct mlxsw_sp_router *router; 9472 9473 /* Flush pending FIB notifications and then flush the device's 9474 * table before requesting another dump. The FIB notification 9475 * block is unregistered, so no need to take RTNL. 9476 */ 9477 mlxsw_core_flush_owq(); 9478 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9479 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9480 } 9481 9482 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9483 struct mlxsw_sp_mp_hash_config { 9484 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 9485 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 9486 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 9487 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 9488 }; 9489 9490 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 9491 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 9492 9493 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 9494 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 9495 9496 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 9497 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 9498 9499 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 9500 { 9501 unsigned long *inner_headers = config->inner_headers; 9502 unsigned long *inner_fields = config->inner_fields; 9503 9504 /* IPv4 inner */ 9505 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9506 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9507 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9508 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9509 /* IPv6 inner */ 9510 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9511 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9512 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9513 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9514 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9515 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9516 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9517 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9518 } 9519 9520 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 9521 { 9522 unsigned long *headers = config->headers; 9523 unsigned long *fields = config->fields; 9524 9525 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 9526 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 9527 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 9528 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 9529 } 9530 9531 static void 9532 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 9533 u32 hash_fields) 9534 { 9535 unsigned long *inner_headers = config->inner_headers; 9536 unsigned long *inner_fields = config->inner_fields; 9537 9538 /* IPv4 Inner */ 9539 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9540 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9541 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 9542 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9543 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 9544 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9545 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9546 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 9547 /* IPv6 inner */ 9548 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9549 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9550 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 9551 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9552 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9553 } 9554 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 9555 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9556 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9557 } 9558 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9559 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9560 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 9561 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9562 /* L4 inner */ 9563 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 9564 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 9565 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 9566 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 9567 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 9568 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 9569 } 9570 9571 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 9572 struct mlxsw_sp_mp_hash_config *config) 9573 { 9574 struct net *net = mlxsw_sp_net(mlxsw_sp); 9575 unsigned long *headers = config->headers; 9576 unsigned long *fields = config->fields; 9577 u32 hash_fields; 9578 9579 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 9580 case 0: 9581 mlxsw_sp_mp4_hash_outer_addr(config); 9582 break; 9583 case 1: 9584 mlxsw_sp_mp4_hash_outer_addr(config); 9585 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 9586 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 9587 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9588 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9589 break; 9590 case 2: 9591 /* Outer */ 9592 mlxsw_sp_mp4_hash_outer_addr(config); 9593 /* Inner */ 9594 mlxsw_sp_mp_hash_inner_l3(config); 9595 break; 9596 case 3: 9597 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; 9598 /* Outer */ 9599 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 9600 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 9601 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 9602 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 9603 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 9604 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 9605 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 9606 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 9607 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 9608 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 9609 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9610 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 9611 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9612 /* Inner */ 9613 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 9614 break; 9615 } 9616 } 9617 9618 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 9619 { 9620 unsigned long *headers = config->headers; 9621 unsigned long *fields = config->fields; 9622 9623 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 9624 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 9625 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 9626 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 9627 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 9628 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 9629 } 9630 9631 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 9632 struct mlxsw_sp_mp_hash_config *config) 9633 { 9634 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 9635 unsigned long *headers = config->headers; 9636 unsigned long *fields = config->fields; 9637 9638 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 9639 case 0: 9640 mlxsw_sp_mp6_hash_outer_addr(config); 9641 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9642 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9643 break; 9644 case 1: 9645 mlxsw_sp_mp6_hash_outer_addr(config); 9646 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 9647 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9648 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9649 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9650 break; 9651 case 2: 9652 /* Outer */ 9653 mlxsw_sp_mp6_hash_outer_addr(config); 9654 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9655 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9656 /* Inner */ 9657 mlxsw_sp_mp_hash_inner_l3(config); 9658 break; 9659 case 3: 9660 /* Outer */ 9661 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 9662 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 9663 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 9664 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 9665 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 9666 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 9667 } 9668 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 9669 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 9670 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 9671 } 9672 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 9673 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9674 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 9675 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9676 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 9677 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9678 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 9679 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9680 /* Inner */ 9681 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 9682 break; 9683 } 9684 } 9685 9686 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9687 { 9688 struct mlxsw_sp_mp_hash_config config = {}; 9689 char recr2_pl[MLXSW_REG_RECR2_LEN]; 9690 unsigned long bit; 9691 u32 seed; 9692 9693 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 9694 mlxsw_reg_recr2_pack(recr2_pl, seed); 9695 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 9696 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 9697 9698 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 9699 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 9700 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 9701 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 9702 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 9703 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 9704 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 9705 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 9706 9707 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 9708 } 9709 #else 9710 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9711 { 9712 return 0; 9713 } 9714 #endif 9715 9716 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 9717 { 9718 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 9719 unsigned int i; 9720 9721 MLXSW_REG_ZERO(rdpm, rdpm_pl); 9722 9723 /* HW is determining switch priority based on DSCP-bits, but the 9724 * kernel is still doing that based on the ToS. Since there's a 9725 * mismatch in bits we need to make sure to translate the right 9726 * value ToS would observe, skipping the 2 least-significant ECN bits. 9727 */ 9728 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 9729 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 9730 9731 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 9732 } 9733 9734 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 9735 { 9736 struct net *net = mlxsw_sp_net(mlxsw_sp); 9737 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 9738 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9739 u64 max_rifs; 9740 9741 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 9742 return -EIO; 9743 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9744 9745 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 9746 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 9747 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 9748 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9749 } 9750 9751 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9752 { 9753 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9754 9755 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 9756 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9757 } 9758 9759 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 9760 .init = mlxsw_sp_router_ll_basic_init, 9761 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 9762 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 9763 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 9764 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 9765 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 9766 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 9767 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 9768 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 9769 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 9770 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 9771 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 9772 }; 9773 9774 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 9775 { 9776 size_t max_size = 0; 9777 int i; 9778 9779 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 9780 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 9781 9782 if (size > max_size) 9783 max_size = size; 9784 } 9785 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 9786 GFP_KERNEL); 9787 if (!router->ll_op_ctx) 9788 return -ENOMEM; 9789 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 9790 return 0; 9791 } 9792 9793 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 9794 { 9795 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 9796 kfree(router->ll_op_ctx); 9797 } 9798 9799 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 9800 { 9801 u16 lb_rif_index; 9802 int err; 9803 9804 /* Create a generic loopback RIF associated with the main table 9805 * (default VRF). Any table can be used, but the main table exists 9806 * anyway, so we do not waste resources. 9807 */ 9808 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 9809 &lb_rif_index); 9810 if (err) 9811 return err; 9812 9813 mlxsw_sp->router->lb_rif_index = lb_rif_index; 9814 9815 return 0; 9816 } 9817 9818 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 9819 { 9820 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 9821 } 9822 9823 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 9824 { 9825 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 9826 9827 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 9828 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 9829 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9830 9831 return 0; 9832 } 9833 9834 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 9835 .init = mlxsw_sp1_router_init, 9836 }; 9837 9838 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 9839 { 9840 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 9841 9842 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 9843 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 9844 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9845 9846 return 0; 9847 } 9848 9849 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 9850 .init = mlxsw_sp2_router_init, 9851 }; 9852 9853 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 9854 struct netlink_ext_ack *extack) 9855 { 9856 struct mlxsw_sp_router *router; 9857 int err; 9858 9859 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 9860 if (!router) 9861 return -ENOMEM; 9862 mutex_init(&router->lock); 9863 mlxsw_sp->router = router; 9864 router->mlxsw_sp = mlxsw_sp; 9865 9866 err = mlxsw_sp->router_ops->init(mlxsw_sp); 9867 if (err) 9868 goto err_router_ops_init; 9869 9870 err = mlxsw_sp_router_xm_init(mlxsw_sp); 9871 if (err) 9872 goto err_xm_init; 9873 9874 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 9875 &mlxsw_sp_router_ll_xm_ops : 9876 &mlxsw_sp_router_ll_basic_ops; 9877 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 9878 9879 err = mlxsw_sp_router_ll_op_ctx_init(router); 9880 if (err) 9881 goto err_ll_op_ctx_init; 9882 9883 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 9884 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 9885 mlxsw_sp_nh_grp_activity_work); 9886 9887 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 9888 err = __mlxsw_sp_router_init(mlxsw_sp); 9889 if (err) 9890 goto err_router_init; 9891 9892 err = mlxsw_sp_rifs_init(mlxsw_sp); 9893 if (err) 9894 goto err_rifs_init; 9895 9896 err = mlxsw_sp_ipips_init(mlxsw_sp); 9897 if (err) 9898 goto err_ipips_init; 9899 9900 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 9901 &mlxsw_sp_nexthop_ht_params); 9902 if (err) 9903 goto err_nexthop_ht_init; 9904 9905 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 9906 &mlxsw_sp_nexthop_group_ht_params); 9907 if (err) 9908 goto err_nexthop_group_ht_init; 9909 9910 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 9911 err = mlxsw_sp_lpm_init(mlxsw_sp); 9912 if (err) 9913 goto err_lpm_init; 9914 9915 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 9916 if (err) 9917 goto err_mr_init; 9918 9919 err = mlxsw_sp_vrs_init(mlxsw_sp); 9920 if (err) 9921 goto err_vrs_init; 9922 9923 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 9924 if (err) 9925 goto err_lb_rif_init; 9926 9927 err = mlxsw_sp_neigh_init(mlxsw_sp); 9928 if (err) 9929 goto err_neigh_init; 9930 9931 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 9932 if (err) 9933 goto err_mp_hash_init; 9934 9935 err = mlxsw_sp_dscp_init(mlxsw_sp); 9936 if (err) 9937 goto err_dscp_init; 9938 9939 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 9940 INIT_LIST_HEAD(&router->fib_event_queue); 9941 spin_lock_init(&router->fib_event_queue_lock); 9942 9943 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 9944 err = register_inetaddr_notifier(&router->inetaddr_nb); 9945 if (err) 9946 goto err_register_inetaddr_notifier; 9947 9948 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 9949 err = register_inet6addr_notifier(&router->inet6addr_nb); 9950 if (err) 9951 goto err_register_inet6addr_notifier; 9952 9953 mlxsw_sp->router->netevent_nb.notifier_call = 9954 mlxsw_sp_router_netevent_event; 9955 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9956 if (err) 9957 goto err_register_netevent_notifier; 9958 9959 mlxsw_sp->router->nexthop_nb.notifier_call = 9960 mlxsw_sp_nexthop_obj_event; 9961 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9962 &mlxsw_sp->router->nexthop_nb, 9963 extack); 9964 if (err) 9965 goto err_register_nexthop_notifier; 9966 9967 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 9968 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 9969 &mlxsw_sp->router->fib_nb, 9970 mlxsw_sp_router_fib_dump_flush, extack); 9971 if (err) 9972 goto err_register_fib_notifier; 9973 9974 return 0; 9975 9976 err_register_fib_notifier: 9977 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 9978 &mlxsw_sp->router->nexthop_nb); 9979 err_register_nexthop_notifier: 9980 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 9981 err_register_netevent_notifier: 9982 unregister_inet6addr_notifier(&router->inet6addr_nb); 9983 err_register_inet6addr_notifier: 9984 unregister_inetaddr_notifier(&router->inetaddr_nb); 9985 err_register_inetaddr_notifier: 9986 mlxsw_core_flush_owq(); 9987 WARN_ON(!list_empty(&router->fib_event_queue)); 9988 err_dscp_init: 9989 err_mp_hash_init: 9990 mlxsw_sp_neigh_fini(mlxsw_sp); 9991 err_neigh_init: 9992 mlxsw_sp_lb_rif_fini(mlxsw_sp); 9993 err_lb_rif_init: 9994 mlxsw_sp_vrs_fini(mlxsw_sp); 9995 err_vrs_init: 9996 mlxsw_sp_mr_fini(mlxsw_sp); 9997 err_mr_init: 9998 mlxsw_sp_lpm_fini(mlxsw_sp); 9999 err_lpm_init: 10000 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10001 err_nexthop_group_ht_init: 10002 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10003 err_nexthop_ht_init: 10004 mlxsw_sp_ipips_fini(mlxsw_sp); 10005 err_ipips_init: 10006 mlxsw_sp_rifs_fini(mlxsw_sp); 10007 err_rifs_init: 10008 __mlxsw_sp_router_fini(mlxsw_sp); 10009 err_router_init: 10010 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10011 mlxsw_sp_router_ll_op_ctx_fini(router); 10012 err_ll_op_ctx_init: 10013 mlxsw_sp_router_xm_fini(mlxsw_sp); 10014 err_xm_init: 10015 err_router_ops_init: 10016 mutex_destroy(&mlxsw_sp->router->lock); 10017 kfree(mlxsw_sp->router); 10018 return err; 10019 } 10020 10021 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10022 { 10023 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10024 &mlxsw_sp->router->fib_nb); 10025 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10026 &mlxsw_sp->router->nexthop_nb); 10027 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10028 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 10029 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 10030 mlxsw_core_flush_owq(); 10031 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 10032 mlxsw_sp_neigh_fini(mlxsw_sp); 10033 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10034 mlxsw_sp_vrs_fini(mlxsw_sp); 10035 mlxsw_sp_mr_fini(mlxsw_sp); 10036 mlxsw_sp_lpm_fini(mlxsw_sp); 10037 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10038 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10039 mlxsw_sp_ipips_fini(mlxsw_sp); 10040 mlxsw_sp_rifs_fini(mlxsw_sp); 10041 __mlxsw_sp_router_fini(mlxsw_sp); 10042 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10043 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 10044 mlxsw_sp_router_xm_fini(mlxsw_sp); 10045 mutex_destroy(&mlxsw_sp->router->lock); 10046 kfree(mlxsw_sp->router); 10047 } 10048