1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/ip_fib.h> 25 #include <net/ip6_fib.h> 26 #include <net/nexthop.h> 27 #include <net/fib_rules.h> 28 #include <net/ip_tunnels.h> 29 #include <net/l3mdev.h> 30 #include <net/addrconf.h> 31 #include <net/ndisc.h> 32 #include <net/ipv6.h> 33 #include <net/fib_notifier.h> 34 #include <net/switchdev.h> 35 36 #include "spectrum.h" 37 #include "core.h" 38 #include "reg.h" 39 #include "spectrum_cnt.h" 40 #include "spectrum_dpipe.h" 41 #include "spectrum_ipip.h" 42 #include "spectrum_mr.h" 43 #include "spectrum_mr_tcam.h" 44 #include "spectrum_router.h" 45 #include "spectrum_span.h" 46 47 struct mlxsw_sp_fib; 48 struct mlxsw_sp_vr; 49 struct mlxsw_sp_lpm_tree; 50 struct mlxsw_sp_rif_ops; 51 52 struct mlxsw_sp_rif { 53 struct list_head nexthop_list; 54 struct list_head neigh_list; 55 struct net_device *dev; /* NULL for underlay RIF */ 56 struct mlxsw_sp_fid *fid; 57 unsigned char addr[ETH_ALEN]; 58 int mtu; 59 u16 rif_index; 60 u16 vr_id; 61 const struct mlxsw_sp_rif_ops *ops; 62 struct mlxsw_sp *mlxsw_sp; 63 64 unsigned int counter_ingress; 65 bool counter_ingress_valid; 66 unsigned int counter_egress; 67 bool counter_egress_valid; 68 }; 69 70 struct mlxsw_sp_rif_params { 71 struct net_device *dev; 72 union { 73 u16 system_port; 74 u16 lag_id; 75 }; 76 u16 vid; 77 bool lag; 78 }; 79 80 struct mlxsw_sp_rif_subport { 81 struct mlxsw_sp_rif common; 82 refcount_t ref_count; 83 union { 84 u16 system_port; 85 u16 lag_id; 86 }; 87 u16 vid; 88 bool lag; 89 }; 90 91 struct mlxsw_sp_rif_ipip_lb { 92 struct mlxsw_sp_rif common; 93 struct mlxsw_sp_rif_ipip_lb_config lb_config; 94 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 95 u16 ul_rif_id; /* Reserved for Spectrum. */ 96 }; 97 98 struct mlxsw_sp_rif_params_ipip_lb { 99 struct mlxsw_sp_rif_params common; 100 struct mlxsw_sp_rif_ipip_lb_config lb_config; 101 }; 102 103 struct mlxsw_sp_rif_ops { 104 enum mlxsw_sp_rif_type type; 105 size_t rif_size; 106 107 void (*setup)(struct mlxsw_sp_rif *rif, 108 const struct mlxsw_sp_rif_params *params); 109 int (*configure)(struct mlxsw_sp_rif *rif); 110 void (*deconfigure)(struct mlxsw_sp_rif *rif); 111 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 114 }; 115 116 struct mlxsw_sp_router_ops { 117 int (*init)(struct mlxsw_sp *mlxsw_sp); 118 }; 119 120 static struct mlxsw_sp_rif * 121 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 122 const struct net_device *dev); 123 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 124 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 125 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 126 struct mlxsw_sp_lpm_tree *lpm_tree); 127 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 128 const struct mlxsw_sp_fib *fib, 129 u8 tree_id); 130 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 131 const struct mlxsw_sp_fib *fib); 132 133 static unsigned int * 134 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 135 enum mlxsw_sp_rif_counter_dir dir) 136 { 137 switch (dir) { 138 case MLXSW_SP_RIF_COUNTER_EGRESS: 139 return &rif->counter_egress; 140 case MLXSW_SP_RIF_COUNTER_INGRESS: 141 return &rif->counter_ingress; 142 } 143 return NULL; 144 } 145 146 static bool 147 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 148 enum mlxsw_sp_rif_counter_dir dir) 149 { 150 switch (dir) { 151 case MLXSW_SP_RIF_COUNTER_EGRESS: 152 return rif->counter_egress_valid; 153 case MLXSW_SP_RIF_COUNTER_INGRESS: 154 return rif->counter_ingress_valid; 155 } 156 return false; 157 } 158 159 static void 160 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 161 enum mlxsw_sp_rif_counter_dir dir, 162 bool valid) 163 { 164 switch (dir) { 165 case MLXSW_SP_RIF_COUNTER_EGRESS: 166 rif->counter_egress_valid = valid; 167 break; 168 case MLXSW_SP_RIF_COUNTER_INGRESS: 169 rif->counter_ingress_valid = valid; 170 break; 171 } 172 } 173 174 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 175 unsigned int counter_index, bool enable, 176 enum mlxsw_sp_rif_counter_dir dir) 177 { 178 char ritr_pl[MLXSW_REG_RITR_LEN]; 179 bool is_egress = false; 180 int err; 181 182 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 183 is_egress = true; 184 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 185 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 186 if (err) 187 return err; 188 189 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 190 is_egress); 191 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 192 } 193 194 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 195 struct mlxsw_sp_rif *rif, 196 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 197 { 198 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 199 unsigned int *p_counter_index; 200 bool valid; 201 int err; 202 203 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 204 if (!valid) 205 return -EINVAL; 206 207 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 208 if (!p_counter_index) 209 return -EINVAL; 210 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 211 MLXSW_REG_RICNT_OPCODE_NOP); 212 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 213 if (err) 214 return err; 215 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 216 return 0; 217 } 218 219 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 220 unsigned int counter_index) 221 { 222 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 223 224 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 225 MLXSW_REG_RICNT_OPCODE_CLEAR); 226 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 227 } 228 229 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp, 230 struct mlxsw_sp_rif *rif, 231 enum mlxsw_sp_rif_counter_dir dir) 232 { 233 unsigned int *p_counter_index; 234 int err; 235 236 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 237 if (!p_counter_index) 238 return -EINVAL; 239 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 240 p_counter_index); 241 if (err) 242 return err; 243 244 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 245 if (err) 246 goto err_counter_clear; 247 248 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 249 *p_counter_index, true, dir); 250 if (err) 251 goto err_counter_edit; 252 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 253 return 0; 254 255 err_counter_edit: 256 err_counter_clear: 257 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 258 *p_counter_index); 259 return err; 260 } 261 262 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp, 263 struct mlxsw_sp_rif *rif, 264 enum mlxsw_sp_rif_counter_dir dir) 265 { 266 unsigned int *p_counter_index; 267 268 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 269 return; 270 271 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 272 if (WARN_ON(!p_counter_index)) 273 return; 274 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 275 *p_counter_index, false, dir); 276 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 277 *p_counter_index); 278 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 279 } 280 281 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 282 { 283 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 284 struct devlink *devlink; 285 286 devlink = priv_to_devlink(mlxsw_sp->core); 287 if (!devlink_dpipe_table_counter_enabled(devlink, 288 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 289 return; 290 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 291 } 292 293 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 294 { 295 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 296 297 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS); 298 } 299 300 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 301 302 struct mlxsw_sp_prefix_usage { 303 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 304 }; 305 306 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 307 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 308 309 static bool 310 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 311 struct mlxsw_sp_prefix_usage *prefix_usage2) 312 { 313 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 314 } 315 316 static void 317 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 318 struct mlxsw_sp_prefix_usage *prefix_usage2) 319 { 320 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 321 } 322 323 static void 324 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 325 unsigned char prefix_len) 326 { 327 set_bit(prefix_len, prefix_usage->b); 328 } 329 330 static void 331 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 332 unsigned char prefix_len) 333 { 334 clear_bit(prefix_len, prefix_usage->b); 335 } 336 337 struct mlxsw_sp_fib_key { 338 unsigned char addr[sizeof(struct in6_addr)]; 339 unsigned char prefix_len; 340 }; 341 342 enum mlxsw_sp_fib_entry_type { 343 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 344 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 345 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 346 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 347 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 348 349 /* This is a special case of local delivery, where a packet should be 350 * decapsulated on reception. Note that there is no corresponding ENCAP, 351 * because that's a type of next hop, not of FIB entry. (There can be 352 * several next hops in a REMOTE entry, and some of them may be 353 * encapsulating entries.) 354 */ 355 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 356 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 357 }; 358 359 struct mlxsw_sp_nexthop_group_info; 360 struct mlxsw_sp_nexthop_group; 361 struct mlxsw_sp_fib_entry; 362 363 struct mlxsw_sp_fib_node { 364 struct mlxsw_sp_fib_entry *fib_entry; 365 struct list_head list; 366 struct rhash_head ht_node; 367 struct mlxsw_sp_fib *fib; 368 struct mlxsw_sp_fib_key key; 369 }; 370 371 struct mlxsw_sp_fib_entry_decap { 372 struct mlxsw_sp_ipip_entry *ipip_entry; 373 u32 tunnel_index; 374 }; 375 376 static struct mlxsw_sp_fib_entry_priv * 377 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 378 { 379 struct mlxsw_sp_fib_entry_priv *priv; 380 381 if (!ll_ops->fib_entry_priv_size) 382 /* No need to have priv */ 383 return NULL; 384 385 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 386 if (!priv) 387 return ERR_PTR(-ENOMEM); 388 refcount_set(&priv->refcnt, 1); 389 return priv; 390 } 391 392 static void 393 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 394 { 395 kfree(priv); 396 } 397 398 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 399 { 400 refcount_inc(&priv->refcnt); 401 } 402 403 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 404 { 405 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 406 return; 407 mlxsw_sp_fib_entry_priv_destroy(priv); 408 } 409 410 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 411 struct mlxsw_sp_fib_entry_priv *priv) 412 { 413 if (!priv) 414 return; 415 mlxsw_sp_fib_entry_priv_hold(priv); 416 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 417 } 418 419 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 420 { 421 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 422 423 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 424 mlxsw_sp_fib_entry_priv_put(priv); 425 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 426 } 427 428 struct mlxsw_sp_fib_entry { 429 struct mlxsw_sp_fib_node *fib_node; 430 enum mlxsw_sp_fib_entry_type type; 431 struct list_head nexthop_group_node; 432 struct mlxsw_sp_nexthop_group *nh_group; 433 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 434 struct mlxsw_sp_fib_entry_priv *priv; 435 }; 436 437 struct mlxsw_sp_fib4_entry { 438 struct mlxsw_sp_fib_entry common; 439 struct fib_info *fi; 440 u32 tb_id; 441 u8 tos; 442 u8 type; 443 }; 444 445 struct mlxsw_sp_fib6_entry { 446 struct mlxsw_sp_fib_entry common; 447 struct list_head rt6_list; 448 unsigned int nrt6; 449 }; 450 451 struct mlxsw_sp_rt6 { 452 struct list_head list; 453 struct fib6_info *rt; 454 }; 455 456 struct mlxsw_sp_lpm_tree { 457 u8 id; /* tree ID */ 458 unsigned int ref_count; 459 enum mlxsw_sp_l3proto proto; 460 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 461 struct mlxsw_sp_prefix_usage prefix_usage; 462 }; 463 464 struct mlxsw_sp_fib { 465 struct rhashtable ht; 466 struct list_head node_list; 467 struct mlxsw_sp_vr *vr; 468 struct mlxsw_sp_lpm_tree *lpm_tree; 469 enum mlxsw_sp_l3proto proto; 470 const struct mlxsw_sp_router_ll_ops *ll_ops; 471 }; 472 473 struct mlxsw_sp_vr { 474 u16 id; /* virtual router ID */ 475 u32 tb_id; /* kernel fib table id */ 476 unsigned int rif_count; 477 struct mlxsw_sp_fib *fib4; 478 struct mlxsw_sp_fib *fib6; 479 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 480 struct mlxsw_sp_rif *ul_rif; 481 refcount_t ul_rif_refcnt; 482 }; 483 484 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 485 enum mlxsw_sp_l3proto proto) 486 { 487 return 0; 488 } 489 490 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 491 { 492 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 493 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 494 } 495 496 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 497 { 498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 499 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 500 } 501 502 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 503 { 504 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 505 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 506 } 507 508 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 509 510 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 511 struct mlxsw_sp_vr *vr, 512 enum mlxsw_sp_l3proto proto) 513 { 514 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 515 struct mlxsw_sp_lpm_tree *lpm_tree; 516 struct mlxsw_sp_fib *fib; 517 int err; 518 519 err = ll_ops->init(mlxsw_sp, vr->id, proto); 520 if (err) 521 return ERR_PTR(err); 522 523 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 524 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 525 if (!fib) 526 return ERR_PTR(-ENOMEM); 527 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 528 if (err) 529 goto err_rhashtable_init; 530 INIT_LIST_HEAD(&fib->node_list); 531 fib->proto = proto; 532 fib->vr = vr; 533 fib->lpm_tree = lpm_tree; 534 fib->ll_ops = ll_ops; 535 mlxsw_sp_lpm_tree_hold(lpm_tree); 536 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 537 if (err) 538 goto err_lpm_tree_bind; 539 return fib; 540 541 err_lpm_tree_bind: 542 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 543 err_rhashtable_init: 544 kfree(fib); 545 return ERR_PTR(err); 546 } 547 548 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 549 struct mlxsw_sp_fib *fib) 550 { 551 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 552 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 553 WARN_ON(!list_empty(&fib->node_list)); 554 rhashtable_destroy(&fib->ht); 555 kfree(fib); 556 } 557 558 static struct mlxsw_sp_lpm_tree * 559 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 560 { 561 static struct mlxsw_sp_lpm_tree *lpm_tree; 562 int i; 563 564 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 565 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 566 if (lpm_tree->ref_count == 0) 567 return lpm_tree; 568 } 569 return NULL; 570 } 571 572 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 573 const struct mlxsw_sp_router_ll_ops *ll_ops, 574 struct mlxsw_sp_lpm_tree *lpm_tree) 575 { 576 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 577 578 mlxsw_reg_xralta_pack(xralta_pl, true, 579 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 580 lpm_tree->id); 581 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 582 } 583 584 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 585 const struct mlxsw_sp_router_ll_ops *ll_ops, 586 struct mlxsw_sp_lpm_tree *lpm_tree) 587 { 588 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 589 590 mlxsw_reg_xralta_pack(xralta_pl, false, 591 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 592 lpm_tree->id); 593 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 594 } 595 596 static int 597 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 598 const struct mlxsw_sp_router_ll_ops *ll_ops, 599 struct mlxsw_sp_prefix_usage *prefix_usage, 600 struct mlxsw_sp_lpm_tree *lpm_tree) 601 { 602 char xralst_pl[MLXSW_REG_XRALST_LEN]; 603 u8 root_bin = 0; 604 u8 prefix; 605 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 606 607 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 608 root_bin = prefix; 609 610 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 611 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 612 if (prefix == 0) 613 continue; 614 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 615 MLXSW_REG_RALST_BIN_NO_CHILD); 616 last_prefix = prefix; 617 } 618 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 619 } 620 621 static struct mlxsw_sp_lpm_tree * 622 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 623 const struct mlxsw_sp_router_ll_ops *ll_ops, 624 struct mlxsw_sp_prefix_usage *prefix_usage, 625 enum mlxsw_sp_l3proto proto) 626 { 627 struct mlxsw_sp_lpm_tree *lpm_tree; 628 int err; 629 630 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 631 if (!lpm_tree) 632 return ERR_PTR(-EBUSY); 633 lpm_tree->proto = proto; 634 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 635 if (err) 636 return ERR_PTR(err); 637 638 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 639 if (err) 640 goto err_left_struct_set; 641 memcpy(&lpm_tree->prefix_usage, prefix_usage, 642 sizeof(lpm_tree->prefix_usage)); 643 memset(&lpm_tree->prefix_ref_count, 0, 644 sizeof(lpm_tree->prefix_ref_count)); 645 lpm_tree->ref_count = 1; 646 return lpm_tree; 647 648 err_left_struct_set: 649 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 650 return ERR_PTR(err); 651 } 652 653 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 654 const struct mlxsw_sp_router_ll_ops *ll_ops, 655 struct mlxsw_sp_lpm_tree *lpm_tree) 656 { 657 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 658 } 659 660 static struct mlxsw_sp_lpm_tree * 661 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 662 struct mlxsw_sp_prefix_usage *prefix_usage, 663 enum mlxsw_sp_l3proto proto) 664 { 665 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 666 struct mlxsw_sp_lpm_tree *lpm_tree; 667 int i; 668 669 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 670 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 671 if (lpm_tree->ref_count != 0 && 672 lpm_tree->proto == proto && 673 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 674 prefix_usage)) { 675 mlxsw_sp_lpm_tree_hold(lpm_tree); 676 return lpm_tree; 677 } 678 } 679 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 680 } 681 682 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 683 { 684 lpm_tree->ref_count++; 685 } 686 687 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 688 struct mlxsw_sp_lpm_tree *lpm_tree) 689 { 690 const struct mlxsw_sp_router_ll_ops *ll_ops = 691 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 692 693 if (--lpm_tree->ref_count == 0) 694 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 695 } 696 697 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 698 699 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 700 { 701 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 702 struct mlxsw_sp_lpm_tree *lpm_tree; 703 u64 max_trees; 704 int err, i; 705 706 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 707 return -EIO; 708 709 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 710 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 711 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 712 sizeof(struct mlxsw_sp_lpm_tree), 713 GFP_KERNEL); 714 if (!mlxsw_sp->router->lpm.trees) 715 return -ENOMEM; 716 717 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 718 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 719 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 720 } 721 722 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 723 MLXSW_SP_L3_PROTO_IPV4); 724 if (IS_ERR(lpm_tree)) { 725 err = PTR_ERR(lpm_tree); 726 goto err_ipv4_tree_get; 727 } 728 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 729 730 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 731 MLXSW_SP_L3_PROTO_IPV6); 732 if (IS_ERR(lpm_tree)) { 733 err = PTR_ERR(lpm_tree); 734 goto err_ipv6_tree_get; 735 } 736 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 737 738 return 0; 739 740 err_ipv6_tree_get: 741 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 742 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 743 err_ipv4_tree_get: 744 kfree(mlxsw_sp->router->lpm.trees); 745 return err; 746 } 747 748 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 749 { 750 struct mlxsw_sp_lpm_tree *lpm_tree; 751 752 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 754 755 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 756 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 757 758 kfree(mlxsw_sp->router->lpm.trees); 759 } 760 761 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 762 { 763 return !!vr->fib4 || !!vr->fib6 || 764 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 765 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 766 } 767 768 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 769 { 770 struct mlxsw_sp_vr *vr; 771 int i; 772 773 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 774 vr = &mlxsw_sp->router->vrs[i]; 775 if (!mlxsw_sp_vr_is_used(vr)) 776 return vr; 777 } 778 return NULL; 779 } 780 781 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 782 const struct mlxsw_sp_fib *fib, u8 tree_id) 783 { 784 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 785 786 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 787 (enum mlxsw_reg_ralxx_protocol) fib->proto, 788 tree_id); 789 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 790 } 791 792 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 793 const struct mlxsw_sp_fib *fib) 794 { 795 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 796 797 /* Bind to tree 0 which is default */ 798 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 799 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 800 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 801 } 802 803 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 804 { 805 /* For our purpose, squash main, default and local tables into one */ 806 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 807 tb_id = RT_TABLE_MAIN; 808 return tb_id; 809 } 810 811 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 812 u32 tb_id) 813 { 814 struct mlxsw_sp_vr *vr; 815 int i; 816 817 tb_id = mlxsw_sp_fix_tb_id(tb_id); 818 819 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 820 vr = &mlxsw_sp->router->vrs[i]; 821 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 822 return vr; 823 } 824 return NULL; 825 } 826 827 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 828 u16 *vr_id) 829 { 830 struct mlxsw_sp_vr *vr; 831 int err = 0; 832 833 mutex_lock(&mlxsw_sp->router->lock); 834 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 835 if (!vr) { 836 err = -ESRCH; 837 goto out; 838 } 839 *vr_id = vr->id; 840 out: 841 mutex_unlock(&mlxsw_sp->router->lock); 842 return err; 843 } 844 845 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 846 enum mlxsw_sp_l3proto proto) 847 { 848 switch (proto) { 849 case MLXSW_SP_L3_PROTO_IPV4: 850 return vr->fib4; 851 case MLXSW_SP_L3_PROTO_IPV6: 852 return vr->fib6; 853 } 854 return NULL; 855 } 856 857 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 858 u32 tb_id, 859 struct netlink_ext_ack *extack) 860 { 861 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 862 struct mlxsw_sp_fib *fib4; 863 struct mlxsw_sp_fib *fib6; 864 struct mlxsw_sp_vr *vr; 865 int err; 866 867 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 868 if (!vr) { 869 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 870 return ERR_PTR(-EBUSY); 871 } 872 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 873 if (IS_ERR(fib4)) 874 return ERR_CAST(fib4); 875 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 876 if (IS_ERR(fib6)) { 877 err = PTR_ERR(fib6); 878 goto err_fib6_create; 879 } 880 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 881 MLXSW_SP_L3_PROTO_IPV4); 882 if (IS_ERR(mr4_table)) { 883 err = PTR_ERR(mr4_table); 884 goto err_mr4_table_create; 885 } 886 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 887 MLXSW_SP_L3_PROTO_IPV6); 888 if (IS_ERR(mr6_table)) { 889 err = PTR_ERR(mr6_table); 890 goto err_mr6_table_create; 891 } 892 893 vr->fib4 = fib4; 894 vr->fib6 = fib6; 895 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 896 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 897 vr->tb_id = tb_id; 898 return vr; 899 900 err_mr6_table_create: 901 mlxsw_sp_mr_table_destroy(mr4_table); 902 err_mr4_table_create: 903 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 904 err_fib6_create: 905 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 906 return ERR_PTR(err); 907 } 908 909 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 910 struct mlxsw_sp_vr *vr) 911 { 912 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 913 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 914 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 915 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 916 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 917 vr->fib6 = NULL; 918 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 919 vr->fib4 = NULL; 920 } 921 922 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 923 struct netlink_ext_ack *extack) 924 { 925 struct mlxsw_sp_vr *vr; 926 927 tb_id = mlxsw_sp_fix_tb_id(tb_id); 928 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 929 if (!vr) 930 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 931 return vr; 932 } 933 934 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 935 { 936 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 937 list_empty(&vr->fib6->node_list) && 938 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 939 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 940 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 941 } 942 943 static bool 944 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 945 enum mlxsw_sp_l3proto proto, u8 tree_id) 946 { 947 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 948 949 if (!mlxsw_sp_vr_is_used(vr)) 950 return false; 951 if (fib->lpm_tree->id == tree_id) 952 return true; 953 return false; 954 } 955 956 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 957 struct mlxsw_sp_fib *fib, 958 struct mlxsw_sp_lpm_tree *new_tree) 959 { 960 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 961 int err; 962 963 fib->lpm_tree = new_tree; 964 mlxsw_sp_lpm_tree_hold(new_tree); 965 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 966 if (err) 967 goto err_tree_bind; 968 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 969 return 0; 970 971 err_tree_bind: 972 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 973 fib->lpm_tree = old_tree; 974 return err; 975 } 976 977 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 978 struct mlxsw_sp_fib *fib, 979 struct mlxsw_sp_lpm_tree *new_tree) 980 { 981 enum mlxsw_sp_l3proto proto = fib->proto; 982 struct mlxsw_sp_lpm_tree *old_tree; 983 u8 old_id, new_id = new_tree->id; 984 struct mlxsw_sp_vr *vr; 985 int i, err; 986 987 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 988 old_id = old_tree->id; 989 990 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 991 vr = &mlxsw_sp->router->vrs[i]; 992 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 993 continue; 994 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 995 mlxsw_sp_vr_fib(vr, proto), 996 new_tree); 997 if (err) 998 goto err_tree_replace; 999 } 1000 1001 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1002 sizeof(new_tree->prefix_ref_count)); 1003 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1004 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1005 1006 return 0; 1007 1008 err_tree_replace: 1009 for (i--; i >= 0; i--) { 1010 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1011 continue; 1012 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1013 mlxsw_sp_vr_fib(vr, proto), 1014 old_tree); 1015 } 1016 return err; 1017 } 1018 1019 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1020 { 1021 struct mlxsw_sp_vr *vr; 1022 u64 max_vrs; 1023 int i; 1024 1025 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1026 return -EIO; 1027 1028 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1029 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1030 GFP_KERNEL); 1031 if (!mlxsw_sp->router->vrs) 1032 return -ENOMEM; 1033 1034 for (i = 0; i < max_vrs; i++) { 1035 vr = &mlxsw_sp->router->vrs[i]; 1036 vr->id = i; 1037 } 1038 1039 return 0; 1040 } 1041 1042 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1043 1044 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1045 { 1046 /* At this stage we're guaranteed not to have new incoming 1047 * FIB notifications and the work queue is free from FIBs 1048 * sitting on top of mlxsw netdevs. However, we can still 1049 * have other FIBs queued. Flush the queue before flushing 1050 * the device's tables. No need for locks, as we're the only 1051 * writer. 1052 */ 1053 mlxsw_core_flush_owq(); 1054 mlxsw_sp_router_fib_flush(mlxsw_sp); 1055 kfree(mlxsw_sp->router->vrs); 1056 } 1057 1058 static struct net_device * 1059 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev) 1060 { 1061 struct ip_tunnel *tun = netdev_priv(ol_dev); 1062 struct net *net = dev_net(ol_dev); 1063 1064 return dev_get_by_index_rcu(net, tun->parms.link); 1065 } 1066 1067 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1068 { 1069 struct net_device *d; 1070 u32 tb_id; 1071 1072 rcu_read_lock(); 1073 d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1074 if (d) 1075 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1076 else 1077 tb_id = RT_TABLE_MAIN; 1078 rcu_read_unlock(); 1079 1080 return tb_id; 1081 } 1082 1083 static struct mlxsw_sp_rif * 1084 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1085 const struct mlxsw_sp_rif_params *params, 1086 struct netlink_ext_ack *extack); 1087 1088 static struct mlxsw_sp_rif_ipip_lb * 1089 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1090 enum mlxsw_sp_ipip_type ipipt, 1091 struct net_device *ol_dev, 1092 struct netlink_ext_ack *extack) 1093 { 1094 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1095 const struct mlxsw_sp_ipip_ops *ipip_ops; 1096 struct mlxsw_sp_rif *rif; 1097 1098 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1099 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1100 .common.dev = ol_dev, 1101 .common.lag = false, 1102 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1103 }; 1104 1105 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1106 if (IS_ERR(rif)) 1107 return ERR_CAST(rif); 1108 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1109 } 1110 1111 static struct mlxsw_sp_ipip_entry * 1112 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1113 enum mlxsw_sp_ipip_type ipipt, 1114 struct net_device *ol_dev) 1115 { 1116 const struct mlxsw_sp_ipip_ops *ipip_ops; 1117 struct mlxsw_sp_ipip_entry *ipip_entry; 1118 struct mlxsw_sp_ipip_entry *ret = NULL; 1119 1120 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1121 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1122 if (!ipip_entry) 1123 return ERR_PTR(-ENOMEM); 1124 1125 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1126 ol_dev, NULL); 1127 if (IS_ERR(ipip_entry->ol_lb)) { 1128 ret = ERR_CAST(ipip_entry->ol_lb); 1129 goto err_ol_ipip_lb_create; 1130 } 1131 1132 ipip_entry->ipipt = ipipt; 1133 ipip_entry->ol_dev = ol_dev; 1134 1135 switch (ipip_ops->ul_proto) { 1136 case MLXSW_SP_L3_PROTO_IPV4: 1137 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev); 1138 break; 1139 case MLXSW_SP_L3_PROTO_IPV6: 1140 WARN_ON(1); 1141 break; 1142 } 1143 1144 return ipip_entry; 1145 1146 err_ol_ipip_lb_create: 1147 kfree(ipip_entry); 1148 return ret; 1149 } 1150 1151 static void 1152 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry) 1153 { 1154 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1155 kfree(ipip_entry); 1156 } 1157 1158 static bool 1159 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1160 const enum mlxsw_sp_l3proto ul_proto, 1161 union mlxsw_sp_l3addr saddr, 1162 u32 ul_tb_id, 1163 struct mlxsw_sp_ipip_entry *ipip_entry) 1164 { 1165 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1166 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1167 union mlxsw_sp_l3addr tun_saddr; 1168 1169 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1170 return false; 1171 1172 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1173 return tun_ul_tb_id == ul_tb_id && 1174 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1175 } 1176 1177 static int 1178 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1179 struct mlxsw_sp_fib_entry *fib_entry, 1180 struct mlxsw_sp_ipip_entry *ipip_entry) 1181 { 1182 u32 tunnel_index; 1183 int err; 1184 1185 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1186 1, &tunnel_index); 1187 if (err) 1188 return err; 1189 1190 ipip_entry->decap_fib_entry = fib_entry; 1191 fib_entry->decap.ipip_entry = ipip_entry; 1192 fib_entry->decap.tunnel_index = tunnel_index; 1193 return 0; 1194 } 1195 1196 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1197 struct mlxsw_sp_fib_entry *fib_entry) 1198 { 1199 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1200 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1201 fib_entry->decap.ipip_entry = NULL; 1202 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1203 1, fib_entry->decap.tunnel_index); 1204 } 1205 1206 static struct mlxsw_sp_fib_node * 1207 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1208 size_t addr_len, unsigned char prefix_len); 1209 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1210 struct mlxsw_sp_fib_entry *fib_entry); 1211 1212 static void 1213 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1214 struct mlxsw_sp_ipip_entry *ipip_entry) 1215 { 1216 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1217 1218 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1219 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1220 1221 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1222 } 1223 1224 static void 1225 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1226 struct mlxsw_sp_ipip_entry *ipip_entry, 1227 struct mlxsw_sp_fib_entry *decap_fib_entry) 1228 { 1229 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1230 ipip_entry)) 1231 return; 1232 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1233 1234 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1235 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1236 } 1237 1238 static struct mlxsw_sp_fib_entry * 1239 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1240 enum mlxsw_sp_l3proto proto, 1241 const union mlxsw_sp_l3addr *addr, 1242 enum mlxsw_sp_fib_entry_type type) 1243 { 1244 struct mlxsw_sp_fib_node *fib_node; 1245 unsigned char addr_prefix_len; 1246 struct mlxsw_sp_fib *fib; 1247 struct mlxsw_sp_vr *vr; 1248 const void *addrp; 1249 size_t addr_len; 1250 u32 addr4; 1251 1252 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1253 if (!vr) 1254 return NULL; 1255 fib = mlxsw_sp_vr_fib(vr, proto); 1256 1257 switch (proto) { 1258 case MLXSW_SP_L3_PROTO_IPV4: 1259 addr4 = be32_to_cpu(addr->addr4); 1260 addrp = &addr4; 1261 addr_len = 4; 1262 addr_prefix_len = 32; 1263 break; 1264 case MLXSW_SP_L3_PROTO_IPV6: 1265 default: 1266 WARN_ON(1); 1267 return NULL; 1268 } 1269 1270 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1271 addr_prefix_len); 1272 if (!fib_node || fib_node->fib_entry->type != type) 1273 return NULL; 1274 1275 return fib_node->fib_entry; 1276 } 1277 1278 /* Given an IPIP entry, find the corresponding decap route. */ 1279 static struct mlxsw_sp_fib_entry * 1280 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1281 struct mlxsw_sp_ipip_entry *ipip_entry) 1282 { 1283 static struct mlxsw_sp_fib_node *fib_node; 1284 const struct mlxsw_sp_ipip_ops *ipip_ops; 1285 unsigned char saddr_prefix_len; 1286 union mlxsw_sp_l3addr saddr; 1287 struct mlxsw_sp_fib *ul_fib; 1288 struct mlxsw_sp_vr *ul_vr; 1289 const void *saddrp; 1290 size_t saddr_len; 1291 u32 ul_tb_id; 1292 u32 saddr4; 1293 1294 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1295 1296 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1297 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1298 if (!ul_vr) 1299 return NULL; 1300 1301 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1302 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1303 ipip_entry->ol_dev); 1304 1305 switch (ipip_ops->ul_proto) { 1306 case MLXSW_SP_L3_PROTO_IPV4: 1307 saddr4 = be32_to_cpu(saddr.addr4); 1308 saddrp = &saddr4; 1309 saddr_len = 4; 1310 saddr_prefix_len = 32; 1311 break; 1312 default: 1313 WARN_ON(1); 1314 return NULL; 1315 } 1316 1317 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1318 saddr_prefix_len); 1319 if (!fib_node || 1320 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1321 return NULL; 1322 1323 return fib_node->fib_entry; 1324 } 1325 1326 static struct mlxsw_sp_ipip_entry * 1327 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1328 enum mlxsw_sp_ipip_type ipipt, 1329 struct net_device *ol_dev) 1330 { 1331 struct mlxsw_sp_ipip_entry *ipip_entry; 1332 1333 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1334 if (IS_ERR(ipip_entry)) 1335 return ipip_entry; 1336 1337 list_add_tail(&ipip_entry->ipip_list_node, 1338 &mlxsw_sp->router->ipip_list); 1339 1340 return ipip_entry; 1341 } 1342 1343 static void 1344 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1345 struct mlxsw_sp_ipip_entry *ipip_entry) 1346 { 1347 list_del(&ipip_entry->ipip_list_node); 1348 mlxsw_sp_ipip_entry_dealloc(ipip_entry); 1349 } 1350 1351 static bool 1352 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1353 const struct net_device *ul_dev, 1354 enum mlxsw_sp_l3proto ul_proto, 1355 union mlxsw_sp_l3addr ul_dip, 1356 struct mlxsw_sp_ipip_entry *ipip_entry) 1357 { 1358 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1359 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1360 1361 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1362 return false; 1363 1364 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1365 ul_tb_id, ipip_entry); 1366 } 1367 1368 /* Given decap parameters, find the corresponding IPIP entry. */ 1369 static struct mlxsw_sp_ipip_entry * 1370 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1371 enum mlxsw_sp_l3proto ul_proto, 1372 union mlxsw_sp_l3addr ul_dip) 1373 { 1374 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1375 struct net_device *ul_dev; 1376 1377 rcu_read_lock(); 1378 1379 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1380 if (!ul_dev) 1381 goto out_unlock; 1382 1383 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1384 ipip_list_node) 1385 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1386 ul_proto, ul_dip, 1387 ipip_entry)) 1388 goto out_unlock; 1389 1390 rcu_read_unlock(); 1391 1392 return NULL; 1393 1394 out_unlock: 1395 rcu_read_unlock(); 1396 return ipip_entry; 1397 } 1398 1399 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1400 const struct net_device *dev, 1401 enum mlxsw_sp_ipip_type *p_type) 1402 { 1403 struct mlxsw_sp_router *router = mlxsw_sp->router; 1404 const struct mlxsw_sp_ipip_ops *ipip_ops; 1405 enum mlxsw_sp_ipip_type ipipt; 1406 1407 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1408 ipip_ops = router->ipip_ops_arr[ipipt]; 1409 if (dev->type == ipip_ops->dev_type) { 1410 if (p_type) 1411 *p_type = ipipt; 1412 return true; 1413 } 1414 } 1415 return false; 1416 } 1417 1418 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1419 const struct net_device *dev) 1420 { 1421 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1422 } 1423 1424 static struct mlxsw_sp_ipip_entry * 1425 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1426 const struct net_device *ol_dev) 1427 { 1428 struct mlxsw_sp_ipip_entry *ipip_entry; 1429 1430 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1431 ipip_list_node) 1432 if (ipip_entry->ol_dev == ol_dev) 1433 return ipip_entry; 1434 1435 return NULL; 1436 } 1437 1438 static struct mlxsw_sp_ipip_entry * 1439 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1440 const struct net_device *ul_dev, 1441 struct mlxsw_sp_ipip_entry *start) 1442 { 1443 struct mlxsw_sp_ipip_entry *ipip_entry; 1444 1445 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1446 ipip_list_node); 1447 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1448 ipip_list_node) { 1449 struct net_device *ol_dev = ipip_entry->ol_dev; 1450 struct net_device *ipip_ul_dev; 1451 1452 rcu_read_lock(); 1453 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1454 rcu_read_unlock(); 1455 1456 if (ipip_ul_dev == ul_dev) 1457 return ipip_entry; 1458 } 1459 1460 return NULL; 1461 } 1462 1463 bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1464 const struct net_device *dev) 1465 { 1466 bool is_ipip_ul; 1467 1468 mutex_lock(&mlxsw_sp->router->lock); 1469 is_ipip_ul = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1470 mutex_unlock(&mlxsw_sp->router->lock); 1471 1472 return is_ipip_ul; 1473 } 1474 1475 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1476 const struct net_device *ol_dev, 1477 enum mlxsw_sp_ipip_type ipipt) 1478 { 1479 const struct mlxsw_sp_ipip_ops *ops 1480 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1481 1482 return ops->can_offload(mlxsw_sp, ol_dev); 1483 } 1484 1485 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1486 struct net_device *ol_dev) 1487 { 1488 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1489 struct mlxsw_sp_ipip_entry *ipip_entry; 1490 enum mlxsw_sp_l3proto ul_proto; 1491 union mlxsw_sp_l3addr saddr; 1492 u32 ul_tb_id; 1493 1494 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1495 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1496 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1497 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1498 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1499 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1500 saddr, ul_tb_id, 1501 NULL)) { 1502 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1503 ol_dev); 1504 if (IS_ERR(ipip_entry)) 1505 return PTR_ERR(ipip_entry); 1506 } 1507 } 1508 1509 return 0; 1510 } 1511 1512 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1513 struct net_device *ol_dev) 1514 { 1515 struct mlxsw_sp_ipip_entry *ipip_entry; 1516 1517 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1518 if (ipip_entry) 1519 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1520 } 1521 1522 static void 1523 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1524 struct mlxsw_sp_ipip_entry *ipip_entry) 1525 { 1526 struct mlxsw_sp_fib_entry *decap_fib_entry; 1527 1528 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1529 if (decap_fib_entry) 1530 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1531 decap_fib_entry); 1532 } 1533 1534 static int 1535 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1536 u16 ul_rif_id, bool enable) 1537 { 1538 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1539 struct mlxsw_sp_rif *rif = &lb_rif->common; 1540 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1541 char ritr_pl[MLXSW_REG_RITR_LEN]; 1542 u32 saddr4; 1543 1544 switch (lb_cf.ul_protocol) { 1545 case MLXSW_SP_L3_PROTO_IPV4: 1546 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1547 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1548 rif->rif_index, rif->vr_id, rif->dev->mtu); 1549 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1550 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET, 1551 ul_vr_id, ul_rif_id, saddr4, lb_cf.okey); 1552 break; 1553 1554 case MLXSW_SP_L3_PROTO_IPV6: 1555 return -EAFNOSUPPORT; 1556 } 1557 1558 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1559 } 1560 1561 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1562 struct net_device *ol_dev) 1563 { 1564 struct mlxsw_sp_ipip_entry *ipip_entry; 1565 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1566 int err = 0; 1567 1568 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1569 if (ipip_entry) { 1570 lb_rif = ipip_entry->ol_lb; 1571 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1572 lb_rif->ul_rif_id, true); 1573 if (err) 1574 goto out; 1575 lb_rif->common.mtu = ol_dev->mtu; 1576 } 1577 1578 out: 1579 return err; 1580 } 1581 1582 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1583 struct net_device *ol_dev) 1584 { 1585 struct mlxsw_sp_ipip_entry *ipip_entry; 1586 1587 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1588 if (ipip_entry) 1589 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1590 } 1591 1592 static void 1593 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1594 struct mlxsw_sp_ipip_entry *ipip_entry) 1595 { 1596 if (ipip_entry->decap_fib_entry) 1597 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1598 } 1599 1600 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1601 struct net_device *ol_dev) 1602 { 1603 struct mlxsw_sp_ipip_entry *ipip_entry; 1604 1605 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1606 if (ipip_entry) 1607 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1608 } 1609 1610 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1611 struct mlxsw_sp_rif *old_rif, 1612 struct mlxsw_sp_rif *new_rif); 1613 static int 1614 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1615 struct mlxsw_sp_ipip_entry *ipip_entry, 1616 bool keep_encap, 1617 struct netlink_ext_ack *extack) 1618 { 1619 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1620 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1621 1622 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1623 ipip_entry->ipipt, 1624 ipip_entry->ol_dev, 1625 extack); 1626 if (IS_ERR(new_lb_rif)) 1627 return PTR_ERR(new_lb_rif); 1628 ipip_entry->ol_lb = new_lb_rif; 1629 1630 if (keep_encap) 1631 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1632 &new_lb_rif->common); 1633 1634 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1635 1636 return 0; 1637 } 1638 1639 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1640 struct mlxsw_sp_rif *rif); 1641 1642 /** 1643 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1644 * @mlxsw_sp: mlxsw_sp. 1645 * @ipip_entry: IPIP entry. 1646 * @recreate_loopback: Recreates the associated loopback RIF. 1647 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1648 * relevant when recreate_loopback is true. 1649 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1650 * is only relevant when recreate_loopback is false. 1651 * @extack: extack. 1652 * 1653 * Return: Non-zero value on failure. 1654 */ 1655 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1656 struct mlxsw_sp_ipip_entry *ipip_entry, 1657 bool recreate_loopback, 1658 bool keep_encap, 1659 bool update_nexthops, 1660 struct netlink_ext_ack *extack) 1661 { 1662 int err; 1663 1664 /* RIFs can't be edited, so to update loopback, we need to destroy and 1665 * recreate it. That creates a window of opportunity where RALUE and 1666 * RATR registers end up referencing a RIF that's already gone. RATRs 1667 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1668 * of RALUE, demote the decap route back. 1669 */ 1670 if (ipip_entry->decap_fib_entry) 1671 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1672 1673 if (recreate_loopback) { 1674 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1675 keep_encap, extack); 1676 if (err) 1677 return err; 1678 } else if (update_nexthops) { 1679 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1680 &ipip_entry->ol_lb->common); 1681 } 1682 1683 if (ipip_entry->ol_dev->flags & IFF_UP) 1684 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1685 1686 return 0; 1687 } 1688 1689 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1690 struct net_device *ol_dev, 1691 struct netlink_ext_ack *extack) 1692 { 1693 struct mlxsw_sp_ipip_entry *ipip_entry = 1694 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1695 1696 if (!ipip_entry) 1697 return 0; 1698 1699 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1700 true, false, false, extack); 1701 } 1702 1703 static int 1704 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1705 struct mlxsw_sp_ipip_entry *ipip_entry, 1706 struct net_device *ul_dev, 1707 bool *demote_this, 1708 struct netlink_ext_ack *extack) 1709 { 1710 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1711 enum mlxsw_sp_l3proto ul_proto; 1712 union mlxsw_sp_l3addr saddr; 1713 1714 /* Moving underlay to a different VRF might cause local address 1715 * conflict, and the conflicting tunnels need to be demoted. 1716 */ 1717 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1718 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1719 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1720 saddr, ul_tb_id, 1721 ipip_entry)) { 1722 *demote_this = true; 1723 return 0; 1724 } 1725 1726 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1727 true, true, false, extack); 1728 } 1729 1730 static int 1731 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1732 struct mlxsw_sp_ipip_entry *ipip_entry, 1733 struct net_device *ul_dev) 1734 { 1735 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1736 false, false, true, NULL); 1737 } 1738 1739 static int 1740 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1741 struct mlxsw_sp_ipip_entry *ipip_entry, 1742 struct net_device *ul_dev) 1743 { 1744 /* A down underlay device causes encapsulated packets to not be 1745 * forwarded, but decap still works. So refresh next hops without 1746 * touching anything else. 1747 */ 1748 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1749 false, false, true, NULL); 1750 } 1751 1752 static int 1753 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1754 struct net_device *ol_dev, 1755 struct netlink_ext_ack *extack) 1756 { 1757 const struct mlxsw_sp_ipip_ops *ipip_ops; 1758 struct mlxsw_sp_ipip_entry *ipip_entry; 1759 int err; 1760 1761 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1762 if (!ipip_entry) 1763 /* A change might make a tunnel eligible for offloading, but 1764 * that is currently not implemented. What falls to slow path 1765 * stays there. 1766 */ 1767 return 0; 1768 1769 /* A change might make a tunnel not eligible for offloading. */ 1770 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1771 ipip_entry->ipipt)) { 1772 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1773 return 0; 1774 } 1775 1776 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1777 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1778 return err; 1779 } 1780 1781 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1782 struct mlxsw_sp_ipip_entry *ipip_entry) 1783 { 1784 struct net_device *ol_dev = ipip_entry->ol_dev; 1785 1786 if (ol_dev->flags & IFF_UP) 1787 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1788 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1789 } 1790 1791 /* The configuration where several tunnels have the same local address in the 1792 * same underlay table needs special treatment in the HW. That is currently not 1793 * implemented in the driver. This function finds and demotes the first tunnel 1794 * with a given source address, except the one passed in in the argument 1795 * `except'. 1796 */ 1797 bool 1798 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1799 enum mlxsw_sp_l3proto ul_proto, 1800 union mlxsw_sp_l3addr saddr, 1801 u32 ul_tb_id, 1802 const struct mlxsw_sp_ipip_entry *except) 1803 { 1804 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1805 1806 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1807 ipip_list_node) { 1808 if (ipip_entry != except && 1809 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1810 ul_tb_id, ipip_entry)) { 1811 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1812 return true; 1813 } 1814 } 1815 1816 return false; 1817 } 1818 1819 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1820 struct net_device *ul_dev) 1821 { 1822 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1823 1824 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1825 ipip_list_node) { 1826 struct net_device *ol_dev = ipip_entry->ol_dev; 1827 struct net_device *ipip_ul_dev; 1828 1829 rcu_read_lock(); 1830 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1831 rcu_read_unlock(); 1832 if (ipip_ul_dev == ul_dev) 1833 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1834 } 1835 } 1836 1837 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1838 struct net_device *ol_dev, 1839 unsigned long event, 1840 struct netdev_notifier_info *info) 1841 { 1842 struct netdev_notifier_changeupper_info *chup; 1843 struct netlink_ext_ack *extack; 1844 int err = 0; 1845 1846 mutex_lock(&mlxsw_sp->router->lock); 1847 switch (event) { 1848 case NETDEV_REGISTER: 1849 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1850 break; 1851 case NETDEV_UNREGISTER: 1852 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1853 break; 1854 case NETDEV_UP: 1855 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1856 break; 1857 case NETDEV_DOWN: 1858 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1859 break; 1860 case NETDEV_CHANGEUPPER: 1861 chup = container_of(info, typeof(*chup), info); 1862 extack = info->extack; 1863 if (netif_is_l3_master(chup->upper_dev)) 1864 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1865 ol_dev, 1866 extack); 1867 break; 1868 case NETDEV_CHANGE: 1869 extack = info->extack; 1870 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1871 ol_dev, extack); 1872 break; 1873 case NETDEV_CHANGEMTU: 1874 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1875 break; 1876 } 1877 mutex_unlock(&mlxsw_sp->router->lock); 1878 return err; 1879 } 1880 1881 static int 1882 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1883 struct mlxsw_sp_ipip_entry *ipip_entry, 1884 struct net_device *ul_dev, 1885 bool *demote_this, 1886 unsigned long event, 1887 struct netdev_notifier_info *info) 1888 { 1889 struct netdev_notifier_changeupper_info *chup; 1890 struct netlink_ext_ack *extack; 1891 1892 switch (event) { 1893 case NETDEV_CHANGEUPPER: 1894 chup = container_of(info, typeof(*chup), info); 1895 extack = info->extack; 1896 if (netif_is_l3_master(chup->upper_dev)) 1897 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1898 ipip_entry, 1899 ul_dev, 1900 demote_this, 1901 extack); 1902 break; 1903 1904 case NETDEV_UP: 1905 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1906 ul_dev); 1907 case NETDEV_DOWN: 1908 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1909 ipip_entry, 1910 ul_dev); 1911 } 1912 return 0; 1913 } 1914 1915 int 1916 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1917 struct net_device *ul_dev, 1918 unsigned long event, 1919 struct netdev_notifier_info *info) 1920 { 1921 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1922 int err = 0; 1923 1924 mutex_lock(&mlxsw_sp->router->lock); 1925 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1926 ul_dev, 1927 ipip_entry))) { 1928 struct mlxsw_sp_ipip_entry *prev; 1929 bool demote_this = false; 1930 1931 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1932 ul_dev, &demote_this, 1933 event, info); 1934 if (err) { 1935 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1936 ul_dev); 1937 break; 1938 } 1939 1940 if (demote_this) { 1941 if (list_is_first(&ipip_entry->ipip_list_node, 1942 &mlxsw_sp->router->ipip_list)) 1943 prev = NULL; 1944 else 1945 /* This can't be cached from previous iteration, 1946 * because that entry could be gone now. 1947 */ 1948 prev = list_prev_entry(ipip_entry, 1949 ipip_list_node); 1950 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1951 ipip_entry = prev; 1952 } 1953 } 1954 mutex_unlock(&mlxsw_sp->router->lock); 1955 1956 return err; 1957 } 1958 1959 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1960 enum mlxsw_sp_l3proto ul_proto, 1961 const union mlxsw_sp_l3addr *ul_sip, 1962 u32 tunnel_index) 1963 { 1964 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1965 struct mlxsw_sp_router *router = mlxsw_sp->router; 1966 struct mlxsw_sp_fib_entry *fib_entry; 1967 int err = 0; 1968 1969 mutex_lock(&mlxsw_sp->router->lock); 1970 1971 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1972 err = -EINVAL; 1973 goto out; 1974 } 1975 1976 router->nve_decap_config.ul_tb_id = ul_tb_id; 1977 router->nve_decap_config.tunnel_index = tunnel_index; 1978 router->nve_decap_config.ul_proto = ul_proto; 1979 router->nve_decap_config.ul_sip = *ul_sip; 1980 router->nve_decap_config.valid = true; 1981 1982 /* It is valid to create a tunnel with a local IP and only later 1983 * assign this IP address to a local interface 1984 */ 1985 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 1986 ul_proto, ul_sip, 1987 type); 1988 if (!fib_entry) 1989 goto out; 1990 1991 fib_entry->decap.tunnel_index = tunnel_index; 1992 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 1993 1994 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1995 if (err) 1996 goto err_fib_entry_update; 1997 1998 goto out; 1999 2000 err_fib_entry_update: 2001 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2002 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2003 out: 2004 mutex_unlock(&mlxsw_sp->router->lock); 2005 return err; 2006 } 2007 2008 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2009 enum mlxsw_sp_l3proto ul_proto, 2010 const union mlxsw_sp_l3addr *ul_sip) 2011 { 2012 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2013 struct mlxsw_sp_router *router = mlxsw_sp->router; 2014 struct mlxsw_sp_fib_entry *fib_entry; 2015 2016 mutex_lock(&mlxsw_sp->router->lock); 2017 2018 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2019 goto out; 2020 2021 router->nve_decap_config.valid = false; 2022 2023 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2024 ul_proto, ul_sip, 2025 type); 2026 if (!fib_entry) 2027 goto out; 2028 2029 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2030 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2031 out: 2032 mutex_unlock(&mlxsw_sp->router->lock); 2033 } 2034 2035 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2036 u32 ul_tb_id, 2037 enum mlxsw_sp_l3proto ul_proto, 2038 const union mlxsw_sp_l3addr *ul_sip) 2039 { 2040 struct mlxsw_sp_router *router = mlxsw_sp->router; 2041 2042 return router->nve_decap_config.valid && 2043 router->nve_decap_config.ul_tb_id == ul_tb_id && 2044 router->nve_decap_config.ul_proto == ul_proto && 2045 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2046 sizeof(*ul_sip)); 2047 } 2048 2049 struct mlxsw_sp_neigh_key { 2050 struct neighbour *n; 2051 }; 2052 2053 struct mlxsw_sp_neigh_entry { 2054 struct list_head rif_list_node; 2055 struct rhash_head ht_node; 2056 struct mlxsw_sp_neigh_key key; 2057 u16 rif; 2058 bool connected; 2059 unsigned char ha[ETH_ALEN]; 2060 struct list_head nexthop_list; /* list of nexthops using 2061 * this neigh entry 2062 */ 2063 struct list_head nexthop_neighs_list_node; 2064 unsigned int counter_index; 2065 bool counter_valid; 2066 }; 2067 2068 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2069 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2070 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2071 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2072 }; 2073 2074 struct mlxsw_sp_neigh_entry * 2075 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2076 struct mlxsw_sp_neigh_entry *neigh_entry) 2077 { 2078 if (!neigh_entry) { 2079 if (list_empty(&rif->neigh_list)) 2080 return NULL; 2081 else 2082 return list_first_entry(&rif->neigh_list, 2083 typeof(*neigh_entry), 2084 rif_list_node); 2085 } 2086 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2087 return NULL; 2088 return list_next_entry(neigh_entry, rif_list_node); 2089 } 2090 2091 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2092 { 2093 return neigh_entry->key.n->tbl->family; 2094 } 2095 2096 unsigned char * 2097 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2098 { 2099 return neigh_entry->ha; 2100 } 2101 2102 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2103 { 2104 struct neighbour *n; 2105 2106 n = neigh_entry->key.n; 2107 return ntohl(*((__be32 *) n->primary_key)); 2108 } 2109 2110 struct in6_addr * 2111 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2112 { 2113 struct neighbour *n; 2114 2115 n = neigh_entry->key.n; 2116 return (struct in6_addr *) &n->primary_key; 2117 } 2118 2119 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2120 struct mlxsw_sp_neigh_entry *neigh_entry, 2121 u64 *p_counter) 2122 { 2123 if (!neigh_entry->counter_valid) 2124 return -EINVAL; 2125 2126 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2127 p_counter, NULL); 2128 } 2129 2130 static struct mlxsw_sp_neigh_entry * 2131 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2132 u16 rif) 2133 { 2134 struct mlxsw_sp_neigh_entry *neigh_entry; 2135 2136 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2137 if (!neigh_entry) 2138 return NULL; 2139 2140 neigh_entry->key.n = n; 2141 neigh_entry->rif = rif; 2142 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2143 2144 return neigh_entry; 2145 } 2146 2147 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2148 { 2149 kfree(neigh_entry); 2150 } 2151 2152 static int 2153 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2154 struct mlxsw_sp_neigh_entry *neigh_entry) 2155 { 2156 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2157 &neigh_entry->ht_node, 2158 mlxsw_sp_neigh_ht_params); 2159 } 2160 2161 static void 2162 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2163 struct mlxsw_sp_neigh_entry *neigh_entry) 2164 { 2165 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2166 &neigh_entry->ht_node, 2167 mlxsw_sp_neigh_ht_params); 2168 } 2169 2170 static bool 2171 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2172 struct mlxsw_sp_neigh_entry *neigh_entry) 2173 { 2174 struct devlink *devlink; 2175 const char *table_name; 2176 2177 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2178 case AF_INET: 2179 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2180 break; 2181 case AF_INET6: 2182 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2183 break; 2184 default: 2185 WARN_ON(1); 2186 return false; 2187 } 2188 2189 devlink = priv_to_devlink(mlxsw_sp->core); 2190 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2191 } 2192 2193 static void 2194 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2195 struct mlxsw_sp_neigh_entry *neigh_entry) 2196 { 2197 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2198 return; 2199 2200 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2201 return; 2202 2203 neigh_entry->counter_valid = true; 2204 } 2205 2206 static void 2207 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2208 struct mlxsw_sp_neigh_entry *neigh_entry) 2209 { 2210 if (!neigh_entry->counter_valid) 2211 return; 2212 mlxsw_sp_flow_counter_free(mlxsw_sp, 2213 neigh_entry->counter_index); 2214 neigh_entry->counter_valid = false; 2215 } 2216 2217 static struct mlxsw_sp_neigh_entry * 2218 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2219 { 2220 struct mlxsw_sp_neigh_entry *neigh_entry; 2221 struct mlxsw_sp_rif *rif; 2222 int err; 2223 2224 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2225 if (!rif) 2226 return ERR_PTR(-EINVAL); 2227 2228 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2229 if (!neigh_entry) 2230 return ERR_PTR(-ENOMEM); 2231 2232 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2233 if (err) 2234 goto err_neigh_entry_insert; 2235 2236 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2237 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2238 2239 return neigh_entry; 2240 2241 err_neigh_entry_insert: 2242 mlxsw_sp_neigh_entry_free(neigh_entry); 2243 return ERR_PTR(err); 2244 } 2245 2246 static void 2247 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2248 struct mlxsw_sp_neigh_entry *neigh_entry) 2249 { 2250 list_del(&neigh_entry->rif_list_node); 2251 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2252 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2253 mlxsw_sp_neigh_entry_free(neigh_entry); 2254 } 2255 2256 static struct mlxsw_sp_neigh_entry * 2257 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2258 { 2259 struct mlxsw_sp_neigh_key key; 2260 2261 key.n = n; 2262 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2263 &key, mlxsw_sp_neigh_ht_params); 2264 } 2265 2266 static void 2267 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2268 { 2269 unsigned long interval; 2270 2271 #if IS_ENABLED(CONFIG_IPV6) 2272 interval = min_t(unsigned long, 2273 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2274 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2275 #else 2276 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2277 #endif 2278 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2279 } 2280 2281 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2282 char *rauhtd_pl, 2283 int ent_index) 2284 { 2285 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2286 struct net_device *dev; 2287 struct neighbour *n; 2288 __be32 dipn; 2289 u32 dip; 2290 u16 rif; 2291 2292 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2293 2294 if (WARN_ON_ONCE(rif >= max_rifs)) 2295 return; 2296 if (!mlxsw_sp->router->rifs[rif]) { 2297 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2298 return; 2299 } 2300 2301 dipn = htonl(dip); 2302 dev = mlxsw_sp->router->rifs[rif]->dev; 2303 n = neigh_lookup(&arp_tbl, &dipn, dev); 2304 if (!n) 2305 return; 2306 2307 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2308 neigh_event_send(n, NULL); 2309 neigh_release(n); 2310 } 2311 2312 #if IS_ENABLED(CONFIG_IPV6) 2313 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2314 char *rauhtd_pl, 2315 int rec_index) 2316 { 2317 struct net_device *dev; 2318 struct neighbour *n; 2319 struct in6_addr dip; 2320 u16 rif; 2321 2322 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2323 (char *) &dip); 2324 2325 if (!mlxsw_sp->router->rifs[rif]) { 2326 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2327 return; 2328 } 2329 2330 dev = mlxsw_sp->router->rifs[rif]->dev; 2331 n = neigh_lookup(&nd_tbl, &dip, dev); 2332 if (!n) 2333 return; 2334 2335 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2336 neigh_event_send(n, NULL); 2337 neigh_release(n); 2338 } 2339 #else 2340 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2341 char *rauhtd_pl, 2342 int rec_index) 2343 { 2344 } 2345 #endif 2346 2347 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2348 char *rauhtd_pl, 2349 int rec_index) 2350 { 2351 u8 num_entries; 2352 int i; 2353 2354 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2355 rec_index); 2356 /* Hardware starts counting at 0, so add 1. */ 2357 num_entries++; 2358 2359 /* Each record consists of several neighbour entries. */ 2360 for (i = 0; i < num_entries; i++) { 2361 int ent_index; 2362 2363 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2364 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2365 ent_index); 2366 } 2367 2368 } 2369 2370 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2371 char *rauhtd_pl, 2372 int rec_index) 2373 { 2374 /* One record contains one entry. */ 2375 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2376 rec_index); 2377 } 2378 2379 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2380 char *rauhtd_pl, int rec_index) 2381 { 2382 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2383 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2384 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2385 rec_index); 2386 break; 2387 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2388 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2389 rec_index); 2390 break; 2391 } 2392 } 2393 2394 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2395 { 2396 u8 num_rec, last_rec_index, num_entries; 2397 2398 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2399 last_rec_index = num_rec - 1; 2400 2401 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2402 return false; 2403 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2404 MLXSW_REG_RAUHTD_TYPE_IPV6) 2405 return true; 2406 2407 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2408 last_rec_index); 2409 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2410 return true; 2411 return false; 2412 } 2413 2414 static int 2415 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2416 char *rauhtd_pl, 2417 enum mlxsw_reg_rauhtd_type type) 2418 { 2419 int i, num_rec; 2420 int err; 2421 2422 /* Ensure the RIF we read from the device does not change mid-dump. */ 2423 mutex_lock(&mlxsw_sp->router->lock); 2424 do { 2425 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2426 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2427 rauhtd_pl); 2428 if (err) { 2429 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2430 break; 2431 } 2432 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2433 for (i = 0; i < num_rec; i++) 2434 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2435 i); 2436 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2437 mutex_unlock(&mlxsw_sp->router->lock); 2438 2439 return err; 2440 } 2441 2442 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2443 { 2444 enum mlxsw_reg_rauhtd_type type; 2445 char *rauhtd_pl; 2446 int err; 2447 2448 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2449 if (!rauhtd_pl) 2450 return -ENOMEM; 2451 2452 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2453 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2454 if (err) 2455 goto out; 2456 2457 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2458 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2459 out: 2460 kfree(rauhtd_pl); 2461 return err; 2462 } 2463 2464 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2465 { 2466 struct mlxsw_sp_neigh_entry *neigh_entry; 2467 2468 mutex_lock(&mlxsw_sp->router->lock); 2469 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2470 nexthop_neighs_list_node) 2471 /* If this neigh have nexthops, make the kernel think this neigh 2472 * is active regardless of the traffic. 2473 */ 2474 neigh_event_send(neigh_entry->key.n, NULL); 2475 mutex_unlock(&mlxsw_sp->router->lock); 2476 } 2477 2478 static void 2479 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2480 { 2481 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2482 2483 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2484 msecs_to_jiffies(interval)); 2485 } 2486 2487 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2488 { 2489 struct mlxsw_sp_router *router; 2490 int err; 2491 2492 router = container_of(work, struct mlxsw_sp_router, 2493 neighs_update.dw.work); 2494 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2495 if (err) 2496 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2497 2498 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2499 2500 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2501 } 2502 2503 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2504 { 2505 struct mlxsw_sp_neigh_entry *neigh_entry; 2506 struct mlxsw_sp_router *router; 2507 2508 router = container_of(work, struct mlxsw_sp_router, 2509 nexthop_probe_dw.work); 2510 /* Iterate over nexthop neighbours, find those who are unresolved and 2511 * send arp on them. This solves the chicken-egg problem when 2512 * the nexthop wouldn't get offloaded until the neighbor is resolved 2513 * but it wouldn't get resolved ever in case traffic is flowing in HW 2514 * using different nexthop. 2515 */ 2516 mutex_lock(&router->lock); 2517 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2518 nexthop_neighs_list_node) 2519 if (!neigh_entry->connected) 2520 neigh_event_send(neigh_entry->key.n, NULL); 2521 mutex_unlock(&router->lock); 2522 2523 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2524 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2525 } 2526 2527 static void 2528 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2529 struct mlxsw_sp_neigh_entry *neigh_entry, 2530 bool removing, bool dead); 2531 2532 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2533 { 2534 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2535 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2536 } 2537 2538 static int 2539 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2540 struct mlxsw_sp_neigh_entry *neigh_entry, 2541 enum mlxsw_reg_rauht_op op) 2542 { 2543 struct neighbour *n = neigh_entry->key.n; 2544 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2545 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2546 2547 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2548 dip); 2549 if (neigh_entry->counter_valid) 2550 mlxsw_reg_rauht_pack_counter(rauht_pl, 2551 neigh_entry->counter_index); 2552 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2553 } 2554 2555 static int 2556 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2557 struct mlxsw_sp_neigh_entry *neigh_entry, 2558 enum mlxsw_reg_rauht_op op) 2559 { 2560 struct neighbour *n = neigh_entry->key.n; 2561 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2562 const char *dip = n->primary_key; 2563 2564 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2565 dip); 2566 if (neigh_entry->counter_valid) 2567 mlxsw_reg_rauht_pack_counter(rauht_pl, 2568 neigh_entry->counter_index); 2569 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2570 } 2571 2572 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2573 { 2574 struct neighbour *n = neigh_entry->key.n; 2575 2576 /* Packets with a link-local destination address are trapped 2577 * after LPM lookup and never reach the neighbour table, so 2578 * there is no need to program such neighbours to the device. 2579 */ 2580 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2581 IPV6_ADDR_LINKLOCAL) 2582 return true; 2583 return false; 2584 } 2585 2586 static void 2587 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2588 struct mlxsw_sp_neigh_entry *neigh_entry, 2589 bool adding) 2590 { 2591 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2592 int err; 2593 2594 if (!adding && !neigh_entry->connected) 2595 return; 2596 neigh_entry->connected = adding; 2597 if (neigh_entry->key.n->tbl->family == AF_INET) { 2598 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2599 op); 2600 if (err) 2601 return; 2602 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2603 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2604 return; 2605 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2606 op); 2607 if (err) 2608 return; 2609 } else { 2610 WARN_ON_ONCE(1); 2611 return; 2612 } 2613 2614 if (adding) 2615 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2616 else 2617 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2618 } 2619 2620 void 2621 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2622 struct mlxsw_sp_neigh_entry *neigh_entry, 2623 bool adding) 2624 { 2625 if (adding) 2626 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2627 else 2628 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2629 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2630 } 2631 2632 struct mlxsw_sp_netevent_work { 2633 struct work_struct work; 2634 struct mlxsw_sp *mlxsw_sp; 2635 struct neighbour *n; 2636 }; 2637 2638 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2639 { 2640 struct mlxsw_sp_netevent_work *net_work = 2641 container_of(work, struct mlxsw_sp_netevent_work, work); 2642 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2643 struct mlxsw_sp_neigh_entry *neigh_entry; 2644 struct neighbour *n = net_work->n; 2645 unsigned char ha[ETH_ALEN]; 2646 bool entry_connected; 2647 u8 nud_state, dead; 2648 2649 /* If these parameters are changed after we release the lock, 2650 * then we are guaranteed to receive another event letting us 2651 * know about it. 2652 */ 2653 read_lock_bh(&n->lock); 2654 memcpy(ha, n->ha, ETH_ALEN); 2655 nud_state = n->nud_state; 2656 dead = n->dead; 2657 read_unlock_bh(&n->lock); 2658 2659 mutex_lock(&mlxsw_sp->router->lock); 2660 mlxsw_sp_span_respin(mlxsw_sp); 2661 2662 entry_connected = nud_state & NUD_VALID && !dead; 2663 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2664 if (!entry_connected && !neigh_entry) 2665 goto out; 2666 if (!neigh_entry) { 2667 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2668 if (IS_ERR(neigh_entry)) 2669 goto out; 2670 } 2671 2672 if (neigh_entry->connected && entry_connected && 2673 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2674 goto out; 2675 2676 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2677 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2678 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2679 dead); 2680 2681 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2682 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2683 2684 out: 2685 mutex_unlock(&mlxsw_sp->router->lock); 2686 neigh_release(n); 2687 kfree(net_work); 2688 } 2689 2690 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2691 2692 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2693 { 2694 struct mlxsw_sp_netevent_work *net_work = 2695 container_of(work, struct mlxsw_sp_netevent_work, work); 2696 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2697 2698 mlxsw_sp_mp_hash_init(mlxsw_sp); 2699 kfree(net_work); 2700 } 2701 2702 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2703 2704 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2705 { 2706 struct mlxsw_sp_netevent_work *net_work = 2707 container_of(work, struct mlxsw_sp_netevent_work, work); 2708 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2709 2710 __mlxsw_sp_router_init(mlxsw_sp); 2711 kfree(net_work); 2712 } 2713 2714 static int mlxsw_sp_router_schedule_work(struct net *net, 2715 struct notifier_block *nb, 2716 void (*cb)(struct work_struct *)) 2717 { 2718 struct mlxsw_sp_netevent_work *net_work; 2719 struct mlxsw_sp_router *router; 2720 2721 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2722 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2723 return NOTIFY_DONE; 2724 2725 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2726 if (!net_work) 2727 return NOTIFY_BAD; 2728 2729 INIT_WORK(&net_work->work, cb); 2730 net_work->mlxsw_sp = router->mlxsw_sp; 2731 mlxsw_core_schedule_work(&net_work->work); 2732 return NOTIFY_DONE; 2733 } 2734 2735 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2736 unsigned long event, void *ptr) 2737 { 2738 struct mlxsw_sp_netevent_work *net_work; 2739 struct mlxsw_sp_port *mlxsw_sp_port; 2740 struct mlxsw_sp *mlxsw_sp; 2741 unsigned long interval; 2742 struct neigh_parms *p; 2743 struct neighbour *n; 2744 2745 switch (event) { 2746 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2747 p = ptr; 2748 2749 /* We don't care about changes in the default table. */ 2750 if (!p->dev || (p->tbl->family != AF_INET && 2751 p->tbl->family != AF_INET6)) 2752 return NOTIFY_DONE; 2753 2754 /* We are in atomic context and can't take RTNL mutex, 2755 * so use RCU variant to walk the device chain. 2756 */ 2757 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2758 if (!mlxsw_sp_port) 2759 return NOTIFY_DONE; 2760 2761 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2762 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2763 mlxsw_sp->router->neighs_update.interval = interval; 2764 2765 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2766 break; 2767 case NETEVENT_NEIGH_UPDATE: 2768 n = ptr; 2769 2770 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2771 return NOTIFY_DONE; 2772 2773 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2774 if (!mlxsw_sp_port) 2775 return NOTIFY_DONE; 2776 2777 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2778 if (!net_work) { 2779 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2780 return NOTIFY_BAD; 2781 } 2782 2783 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2784 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2785 net_work->n = n; 2786 2787 /* Take a reference to ensure the neighbour won't be 2788 * destructed until we drop the reference in delayed 2789 * work. 2790 */ 2791 neigh_clone(n); 2792 mlxsw_core_schedule_work(&net_work->work); 2793 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2794 break; 2795 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2796 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2797 return mlxsw_sp_router_schedule_work(ptr, nb, 2798 mlxsw_sp_router_mp_hash_event_work); 2799 2800 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2801 return mlxsw_sp_router_schedule_work(ptr, nb, 2802 mlxsw_sp_router_update_priority_work); 2803 } 2804 2805 return NOTIFY_DONE; 2806 } 2807 2808 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2809 { 2810 int err; 2811 2812 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2813 &mlxsw_sp_neigh_ht_params); 2814 if (err) 2815 return err; 2816 2817 /* Initialize the polling interval according to the default 2818 * table. 2819 */ 2820 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2821 2822 /* Create the delayed works for the activity_update */ 2823 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2824 mlxsw_sp_router_neighs_update_work); 2825 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2826 mlxsw_sp_router_probe_unresolved_nexthops); 2827 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2828 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2829 return 0; 2830 } 2831 2832 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2833 { 2834 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2835 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2836 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2837 } 2838 2839 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2840 struct mlxsw_sp_rif *rif) 2841 { 2842 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2843 2844 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2845 rif_list_node) { 2846 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2847 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2848 } 2849 } 2850 2851 enum mlxsw_sp_nexthop_type { 2852 MLXSW_SP_NEXTHOP_TYPE_ETH, 2853 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2854 }; 2855 2856 enum mlxsw_sp_nexthop_action { 2857 /* Nexthop forwards packets to an egress RIF */ 2858 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2859 /* Nexthop discards packets */ 2860 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2861 /* Nexthop traps packets */ 2862 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2863 }; 2864 2865 struct mlxsw_sp_nexthop_key { 2866 struct fib_nh *fib_nh; 2867 }; 2868 2869 struct mlxsw_sp_nexthop { 2870 struct list_head neigh_list_node; /* member of neigh entry list */ 2871 struct list_head rif_list_node; 2872 struct list_head router_list_node; 2873 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2874 * this nexthop belongs to 2875 */ 2876 struct rhash_head ht_node; 2877 struct neigh_table *neigh_tbl; 2878 struct mlxsw_sp_nexthop_key key; 2879 unsigned char gw_addr[sizeof(struct in6_addr)]; 2880 int ifindex; 2881 int nh_weight; 2882 int norm_nh_weight; 2883 int num_adj_entries; 2884 struct mlxsw_sp_rif *rif; 2885 u8 should_offload:1, /* set indicates this nexthop should be written 2886 * to the adjacency table. 2887 */ 2888 offloaded:1, /* set indicates this nexthop was written to the 2889 * adjacency table. 2890 */ 2891 update:1; /* set indicates this nexthop should be updated in the 2892 * adjacency table (f.e., its MAC changed). 2893 */ 2894 enum mlxsw_sp_nexthop_action action; 2895 enum mlxsw_sp_nexthop_type type; 2896 union { 2897 struct mlxsw_sp_neigh_entry *neigh_entry; 2898 struct mlxsw_sp_ipip_entry *ipip_entry; 2899 }; 2900 unsigned int counter_index; 2901 bool counter_valid; 2902 }; 2903 2904 enum mlxsw_sp_nexthop_group_type { 2905 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2906 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2907 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2908 }; 2909 2910 struct mlxsw_sp_nexthop_group_info { 2911 struct mlxsw_sp_nexthop_group *nh_grp; 2912 u32 adj_index; 2913 u16 ecmp_size; 2914 u16 count; 2915 int sum_norm_weight; 2916 u8 adj_index_valid:1, 2917 gateway:1, /* routes using the group use a gateway */ 2918 is_resilient:1; 2919 struct list_head list; /* member in nh_res_grp_list */ 2920 struct mlxsw_sp_nexthop nexthops[0]; 2921 #define nh_rif nexthops[0].rif 2922 }; 2923 2924 struct mlxsw_sp_nexthop_group_vr_key { 2925 u16 vr_id; 2926 enum mlxsw_sp_l3proto proto; 2927 }; 2928 2929 struct mlxsw_sp_nexthop_group_vr_entry { 2930 struct list_head list; /* member in vr_list */ 2931 struct rhash_head ht_node; /* member in vr_ht */ 2932 refcount_t ref_count; 2933 struct mlxsw_sp_nexthop_group_vr_key key; 2934 }; 2935 2936 struct mlxsw_sp_nexthop_group { 2937 struct rhash_head ht_node; 2938 struct list_head fib_list; /* list of fib entries that use this group */ 2939 union { 2940 struct { 2941 struct fib_info *fi; 2942 } ipv4; 2943 struct { 2944 u32 id; 2945 } obj; 2946 }; 2947 struct mlxsw_sp_nexthop_group_info *nhgi; 2948 struct list_head vr_list; 2949 struct rhashtable vr_ht; 2950 enum mlxsw_sp_nexthop_group_type type; 2951 bool can_destroy; 2952 }; 2953 2954 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2955 struct mlxsw_sp_nexthop *nh) 2956 { 2957 struct devlink *devlink; 2958 2959 devlink = priv_to_devlink(mlxsw_sp->core); 2960 if (!devlink_dpipe_table_counter_enabled(devlink, 2961 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2962 return; 2963 2964 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2965 return; 2966 2967 nh->counter_valid = true; 2968 } 2969 2970 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 2971 struct mlxsw_sp_nexthop *nh) 2972 { 2973 if (!nh->counter_valid) 2974 return; 2975 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 2976 nh->counter_valid = false; 2977 } 2978 2979 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 2980 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 2981 { 2982 if (!nh->counter_valid) 2983 return -EINVAL; 2984 2985 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 2986 p_counter, NULL); 2987 } 2988 2989 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 2990 struct mlxsw_sp_nexthop *nh) 2991 { 2992 if (!nh) { 2993 if (list_empty(&router->nexthop_list)) 2994 return NULL; 2995 else 2996 return list_first_entry(&router->nexthop_list, 2997 typeof(*nh), router_list_node); 2998 } 2999 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3000 return NULL; 3001 return list_next_entry(nh, router_list_node); 3002 } 3003 3004 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3005 { 3006 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3007 } 3008 3009 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3010 { 3011 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3012 !mlxsw_sp_nexthop_is_forward(nh)) 3013 return NULL; 3014 return nh->neigh_entry->ha; 3015 } 3016 3017 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3018 u32 *p_adj_size, u32 *p_adj_hash_index) 3019 { 3020 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3021 u32 adj_hash_index = 0; 3022 int i; 3023 3024 if (!nh->offloaded || !nhgi->adj_index_valid) 3025 return -EINVAL; 3026 3027 *p_adj_index = nhgi->adj_index; 3028 *p_adj_size = nhgi->ecmp_size; 3029 3030 for (i = 0; i < nhgi->count; i++) { 3031 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3032 3033 if (nh_iter == nh) 3034 break; 3035 if (nh_iter->offloaded) 3036 adj_hash_index += nh_iter->num_adj_entries; 3037 } 3038 3039 *p_adj_hash_index = adj_hash_index; 3040 return 0; 3041 } 3042 3043 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3044 { 3045 return nh->rif; 3046 } 3047 3048 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3049 { 3050 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3051 int i; 3052 3053 for (i = 0; i < nhgi->count; i++) { 3054 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3055 3056 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3057 return true; 3058 } 3059 return false; 3060 } 3061 3062 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3063 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3064 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3065 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3066 .automatic_shrinking = true, 3067 }; 3068 3069 static struct mlxsw_sp_nexthop_group_vr_entry * 3070 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3071 const struct mlxsw_sp_fib *fib) 3072 { 3073 struct mlxsw_sp_nexthop_group_vr_key key; 3074 3075 memset(&key, 0, sizeof(key)); 3076 key.vr_id = fib->vr->id; 3077 key.proto = fib->proto; 3078 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3079 mlxsw_sp_nexthop_group_vr_ht_params); 3080 } 3081 3082 static int 3083 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3084 const struct mlxsw_sp_fib *fib) 3085 { 3086 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3087 int err; 3088 3089 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3090 if (!vr_entry) 3091 return -ENOMEM; 3092 3093 vr_entry->key.vr_id = fib->vr->id; 3094 vr_entry->key.proto = fib->proto; 3095 refcount_set(&vr_entry->ref_count, 1); 3096 3097 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3098 mlxsw_sp_nexthop_group_vr_ht_params); 3099 if (err) 3100 goto err_hashtable_insert; 3101 3102 list_add(&vr_entry->list, &nh_grp->vr_list); 3103 3104 return 0; 3105 3106 err_hashtable_insert: 3107 kfree(vr_entry); 3108 return err; 3109 } 3110 3111 static void 3112 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3113 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3114 { 3115 list_del(&vr_entry->list); 3116 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3117 mlxsw_sp_nexthop_group_vr_ht_params); 3118 kfree(vr_entry); 3119 } 3120 3121 static int 3122 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3123 const struct mlxsw_sp_fib *fib) 3124 { 3125 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3126 3127 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3128 if (vr_entry) { 3129 refcount_inc(&vr_entry->ref_count); 3130 return 0; 3131 } 3132 3133 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3134 } 3135 3136 static void 3137 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3138 const struct mlxsw_sp_fib *fib) 3139 { 3140 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3141 3142 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3143 if (WARN_ON_ONCE(!vr_entry)) 3144 return; 3145 3146 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3147 return; 3148 3149 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3150 } 3151 3152 struct mlxsw_sp_nexthop_group_cmp_arg { 3153 enum mlxsw_sp_nexthop_group_type type; 3154 union { 3155 struct fib_info *fi; 3156 struct mlxsw_sp_fib6_entry *fib6_entry; 3157 u32 id; 3158 }; 3159 }; 3160 3161 static bool 3162 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3163 const struct in6_addr *gw, int ifindex, 3164 int weight) 3165 { 3166 int i; 3167 3168 for (i = 0; i < nh_grp->nhgi->count; i++) { 3169 const struct mlxsw_sp_nexthop *nh; 3170 3171 nh = &nh_grp->nhgi->nexthops[i]; 3172 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3173 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3174 return true; 3175 } 3176 3177 return false; 3178 } 3179 3180 static bool 3181 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3182 const struct mlxsw_sp_fib6_entry *fib6_entry) 3183 { 3184 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3185 3186 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3187 return false; 3188 3189 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3190 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3191 struct in6_addr *gw; 3192 int ifindex, weight; 3193 3194 ifindex = fib6_nh->fib_nh_dev->ifindex; 3195 weight = fib6_nh->fib_nh_weight; 3196 gw = &fib6_nh->fib_nh_gw6; 3197 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3198 weight)) 3199 return false; 3200 } 3201 3202 return true; 3203 } 3204 3205 static int 3206 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3207 { 3208 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3209 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3210 3211 if (nh_grp->type != cmp_arg->type) 3212 return 1; 3213 3214 switch (cmp_arg->type) { 3215 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3216 return cmp_arg->fi != nh_grp->ipv4.fi; 3217 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3218 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3219 cmp_arg->fib6_entry); 3220 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3221 return cmp_arg->id != nh_grp->obj.id; 3222 default: 3223 WARN_ON(1); 3224 return 1; 3225 } 3226 } 3227 3228 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3229 { 3230 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3231 const struct mlxsw_sp_nexthop *nh; 3232 struct fib_info *fi; 3233 unsigned int val; 3234 int i; 3235 3236 switch (nh_grp->type) { 3237 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3238 fi = nh_grp->ipv4.fi; 3239 return jhash(&fi, sizeof(fi), seed); 3240 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3241 val = nh_grp->nhgi->count; 3242 for (i = 0; i < nh_grp->nhgi->count; i++) { 3243 nh = &nh_grp->nhgi->nexthops[i]; 3244 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3245 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3246 } 3247 return jhash(&val, sizeof(val), seed); 3248 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3249 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3250 default: 3251 WARN_ON(1); 3252 return 0; 3253 } 3254 } 3255 3256 static u32 3257 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3258 { 3259 unsigned int val = fib6_entry->nrt6; 3260 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3261 3262 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3263 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3264 struct net_device *dev = fib6_nh->fib_nh_dev; 3265 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3266 3267 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3268 val ^= jhash(gw, sizeof(*gw), seed); 3269 } 3270 3271 return jhash(&val, sizeof(val), seed); 3272 } 3273 3274 static u32 3275 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3276 { 3277 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3278 3279 switch (cmp_arg->type) { 3280 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3281 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3282 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3283 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3284 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3285 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3286 default: 3287 WARN_ON(1); 3288 return 0; 3289 } 3290 } 3291 3292 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3293 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3294 .hashfn = mlxsw_sp_nexthop_group_hash, 3295 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3296 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3297 }; 3298 3299 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3300 struct mlxsw_sp_nexthop_group *nh_grp) 3301 { 3302 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3303 !nh_grp->nhgi->gateway) 3304 return 0; 3305 3306 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3307 &nh_grp->ht_node, 3308 mlxsw_sp_nexthop_group_ht_params); 3309 } 3310 3311 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3312 struct mlxsw_sp_nexthop_group *nh_grp) 3313 { 3314 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3315 !nh_grp->nhgi->gateway) 3316 return; 3317 3318 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3319 &nh_grp->ht_node, 3320 mlxsw_sp_nexthop_group_ht_params); 3321 } 3322 3323 static struct mlxsw_sp_nexthop_group * 3324 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3325 struct fib_info *fi) 3326 { 3327 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3328 3329 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3330 cmp_arg.fi = fi; 3331 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3332 &cmp_arg, 3333 mlxsw_sp_nexthop_group_ht_params); 3334 } 3335 3336 static struct mlxsw_sp_nexthop_group * 3337 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3338 struct mlxsw_sp_fib6_entry *fib6_entry) 3339 { 3340 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3341 3342 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3343 cmp_arg.fib6_entry = fib6_entry; 3344 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3345 &cmp_arg, 3346 mlxsw_sp_nexthop_group_ht_params); 3347 } 3348 3349 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3350 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3351 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3352 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3353 }; 3354 3355 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3356 struct mlxsw_sp_nexthop *nh) 3357 { 3358 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3359 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3360 } 3361 3362 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3363 struct mlxsw_sp_nexthop *nh) 3364 { 3365 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3366 mlxsw_sp_nexthop_ht_params); 3367 } 3368 3369 static struct mlxsw_sp_nexthop * 3370 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3371 struct mlxsw_sp_nexthop_key key) 3372 { 3373 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3374 mlxsw_sp_nexthop_ht_params); 3375 } 3376 3377 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3378 enum mlxsw_sp_l3proto proto, 3379 u16 vr_id, 3380 u32 adj_index, u16 ecmp_size, 3381 u32 new_adj_index, 3382 u16 new_ecmp_size) 3383 { 3384 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3385 3386 mlxsw_reg_raleu_pack(raleu_pl, 3387 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3388 adj_index, ecmp_size, new_adj_index, 3389 new_ecmp_size); 3390 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3391 } 3392 3393 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3394 struct mlxsw_sp_nexthop_group *nh_grp, 3395 u32 old_adj_index, u16 old_ecmp_size) 3396 { 3397 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3398 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3399 int err; 3400 3401 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3402 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3403 vr_entry->key.proto, 3404 vr_entry->key.vr_id, 3405 old_adj_index, 3406 old_ecmp_size, 3407 nhgi->adj_index, 3408 nhgi->ecmp_size); 3409 if (err) 3410 goto err_mass_update_vr; 3411 } 3412 return 0; 3413 3414 err_mass_update_vr: 3415 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3416 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3417 vr_entry->key.vr_id, 3418 nhgi->adj_index, 3419 nhgi->ecmp_size, 3420 old_adj_index, old_ecmp_size); 3421 return err; 3422 } 3423 3424 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3425 u32 adj_index, 3426 struct mlxsw_sp_nexthop *nh, 3427 bool force, char *ratr_pl) 3428 { 3429 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3430 enum mlxsw_reg_ratr_op op; 3431 u16 rif_index; 3432 3433 rif_index = nh->rif ? nh->rif->rif_index : 3434 mlxsw_sp->router->lb_rif_index; 3435 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3436 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3437 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3438 adj_index, rif_index); 3439 switch (nh->action) { 3440 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3441 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3442 break; 3443 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3444 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3445 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3446 break; 3447 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3448 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3449 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3450 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3451 break; 3452 default: 3453 WARN_ON_ONCE(1); 3454 return -EINVAL; 3455 } 3456 if (nh->counter_valid) 3457 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3458 else 3459 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3460 3461 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3462 } 3463 3464 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3465 struct mlxsw_sp_nexthop *nh, bool force, 3466 char *ratr_pl) 3467 { 3468 int i; 3469 3470 for (i = 0; i < nh->num_adj_entries; i++) { 3471 int err; 3472 3473 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3474 nh, force, ratr_pl); 3475 if (err) 3476 return err; 3477 } 3478 3479 return 0; 3480 } 3481 3482 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3483 u32 adj_index, 3484 struct mlxsw_sp_nexthop *nh, 3485 bool force, char *ratr_pl) 3486 { 3487 const struct mlxsw_sp_ipip_ops *ipip_ops; 3488 3489 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3490 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3491 force, ratr_pl); 3492 } 3493 3494 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3495 u32 adj_index, 3496 struct mlxsw_sp_nexthop *nh, bool force, 3497 char *ratr_pl) 3498 { 3499 int i; 3500 3501 for (i = 0; i < nh->num_adj_entries; i++) { 3502 int err; 3503 3504 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3505 nh, force, ratr_pl); 3506 if (err) 3507 return err; 3508 } 3509 3510 return 0; 3511 } 3512 3513 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3514 struct mlxsw_sp_nexthop *nh, bool force, 3515 char *ratr_pl) 3516 { 3517 /* When action is discard or trap, the nexthop must be 3518 * programmed as an Ethernet nexthop. 3519 */ 3520 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3521 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3522 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3523 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3524 force, ratr_pl); 3525 else 3526 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3527 force, ratr_pl); 3528 } 3529 3530 static int 3531 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3532 struct mlxsw_sp_nexthop_group_info *nhgi, 3533 bool reallocate) 3534 { 3535 char ratr_pl[MLXSW_REG_RATR_LEN]; 3536 u32 adj_index = nhgi->adj_index; /* base */ 3537 struct mlxsw_sp_nexthop *nh; 3538 int i; 3539 3540 for (i = 0; i < nhgi->count; i++) { 3541 nh = &nhgi->nexthops[i]; 3542 3543 if (!nh->should_offload) { 3544 nh->offloaded = 0; 3545 continue; 3546 } 3547 3548 if (nh->update || reallocate) { 3549 int err = 0; 3550 3551 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3552 true, ratr_pl); 3553 if (err) 3554 return err; 3555 nh->update = 0; 3556 nh->offloaded = 1; 3557 } 3558 adj_index += nh->num_adj_entries; 3559 } 3560 return 0; 3561 } 3562 3563 static int 3564 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3565 struct mlxsw_sp_nexthop_group *nh_grp) 3566 { 3567 struct mlxsw_sp_fib_entry *fib_entry; 3568 int err; 3569 3570 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3571 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3572 if (err) 3573 return err; 3574 } 3575 return 0; 3576 } 3577 3578 struct mlxsw_sp_adj_grp_size_range { 3579 u16 start; /* Inclusive */ 3580 u16 end; /* Inclusive */ 3581 }; 3582 3583 /* Ordered by range start value */ 3584 static const struct mlxsw_sp_adj_grp_size_range 3585 mlxsw_sp1_adj_grp_size_ranges[] = { 3586 { .start = 1, .end = 64 }, 3587 { .start = 512, .end = 512 }, 3588 { .start = 1024, .end = 1024 }, 3589 { .start = 2048, .end = 2048 }, 3590 { .start = 4096, .end = 4096 }, 3591 }; 3592 3593 /* Ordered by range start value */ 3594 static const struct mlxsw_sp_adj_grp_size_range 3595 mlxsw_sp2_adj_grp_size_ranges[] = { 3596 { .start = 1, .end = 128 }, 3597 { .start = 256, .end = 256 }, 3598 { .start = 512, .end = 512 }, 3599 { .start = 1024, .end = 1024 }, 3600 { .start = 2048, .end = 2048 }, 3601 { .start = 4096, .end = 4096 }, 3602 }; 3603 3604 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3605 u16 *p_adj_grp_size) 3606 { 3607 int i; 3608 3609 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3610 const struct mlxsw_sp_adj_grp_size_range *size_range; 3611 3612 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3613 3614 if (*p_adj_grp_size >= size_range->start && 3615 *p_adj_grp_size <= size_range->end) 3616 return; 3617 3618 if (*p_adj_grp_size <= size_range->end) { 3619 *p_adj_grp_size = size_range->end; 3620 return; 3621 } 3622 } 3623 } 3624 3625 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3626 u16 *p_adj_grp_size, 3627 unsigned int alloc_size) 3628 { 3629 int i; 3630 3631 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3632 const struct mlxsw_sp_adj_grp_size_range *size_range; 3633 3634 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3635 3636 if (alloc_size >= size_range->end) { 3637 *p_adj_grp_size = size_range->end; 3638 return; 3639 } 3640 } 3641 } 3642 3643 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3644 u16 *p_adj_grp_size) 3645 { 3646 unsigned int alloc_size; 3647 int err; 3648 3649 /* Round up the requested group size to the next size supported 3650 * by the device and make sure the request can be satisfied. 3651 */ 3652 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3653 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3654 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3655 *p_adj_grp_size, &alloc_size); 3656 if (err) 3657 return err; 3658 /* It is possible the allocation results in more allocated 3659 * entries than requested. Try to use as much of them as 3660 * possible. 3661 */ 3662 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3663 3664 return 0; 3665 } 3666 3667 static void 3668 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3669 { 3670 int i, g = 0, sum_norm_weight = 0; 3671 struct mlxsw_sp_nexthop *nh; 3672 3673 for (i = 0; i < nhgi->count; i++) { 3674 nh = &nhgi->nexthops[i]; 3675 3676 if (!nh->should_offload) 3677 continue; 3678 if (g > 0) 3679 g = gcd(nh->nh_weight, g); 3680 else 3681 g = nh->nh_weight; 3682 } 3683 3684 for (i = 0; i < nhgi->count; i++) { 3685 nh = &nhgi->nexthops[i]; 3686 3687 if (!nh->should_offload) 3688 continue; 3689 nh->norm_nh_weight = nh->nh_weight / g; 3690 sum_norm_weight += nh->norm_nh_weight; 3691 } 3692 3693 nhgi->sum_norm_weight = sum_norm_weight; 3694 } 3695 3696 static void 3697 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3698 { 3699 int i, weight = 0, lower_bound = 0; 3700 int total = nhgi->sum_norm_weight; 3701 u16 ecmp_size = nhgi->ecmp_size; 3702 3703 for (i = 0; i < nhgi->count; i++) { 3704 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3705 int upper_bound; 3706 3707 if (!nh->should_offload) 3708 continue; 3709 weight += nh->norm_nh_weight; 3710 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3711 nh->num_adj_entries = upper_bound - lower_bound; 3712 lower_bound = upper_bound; 3713 } 3714 } 3715 3716 static struct mlxsw_sp_nexthop * 3717 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3718 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3719 3720 static void 3721 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3722 struct mlxsw_sp_nexthop_group *nh_grp) 3723 { 3724 int i; 3725 3726 for (i = 0; i < nh_grp->nhgi->count; i++) { 3727 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3728 3729 if (nh->offloaded) 3730 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3731 else 3732 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3733 } 3734 } 3735 3736 static void 3737 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3738 struct mlxsw_sp_fib6_entry *fib6_entry) 3739 { 3740 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3741 3742 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3743 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3744 struct mlxsw_sp_nexthop *nh; 3745 3746 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3747 if (nh && nh->offloaded) 3748 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3749 else 3750 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3751 } 3752 } 3753 3754 static void 3755 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3756 struct mlxsw_sp_nexthop_group *nh_grp) 3757 { 3758 struct mlxsw_sp_fib6_entry *fib6_entry; 3759 3760 /* Unfortunately, in IPv6 the route and the nexthop are described by 3761 * the same struct, so we need to iterate over all the routes using the 3762 * nexthop group and set / clear the offload indication for them. 3763 */ 3764 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3765 common.nexthop_group_node) 3766 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3767 } 3768 3769 static void 3770 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3771 const struct mlxsw_sp_nexthop *nh, 3772 u16 bucket_index) 3773 { 3774 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3775 bool offload = false, trap = false; 3776 3777 if (nh->offloaded) { 3778 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3779 trap = true; 3780 else 3781 offload = true; 3782 } 3783 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3784 bucket_index, offload, trap); 3785 } 3786 3787 static void 3788 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3789 struct mlxsw_sp_nexthop_group *nh_grp) 3790 { 3791 int i; 3792 3793 /* Do not update the flags if the nexthop group is being destroyed 3794 * since: 3795 * 1. The nexthop objects is being deleted, in which case the flags are 3796 * irrelevant. 3797 * 2. The nexthop group was replaced by a newer group, in which case 3798 * the flags of the nexthop object were already updated based on the 3799 * new group. 3800 */ 3801 if (nh_grp->can_destroy) 3802 return; 3803 3804 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3805 nh_grp->nhgi->adj_index_valid, false); 3806 3807 /* Update flags of individual nexthop buckets in case of a resilient 3808 * nexthop group. 3809 */ 3810 if (!nh_grp->nhgi->is_resilient) 3811 return; 3812 3813 for (i = 0; i < nh_grp->nhgi->count; i++) { 3814 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3815 3816 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3817 } 3818 } 3819 3820 static void 3821 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3822 struct mlxsw_sp_nexthop_group *nh_grp) 3823 { 3824 switch (nh_grp->type) { 3825 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3826 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3827 break; 3828 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3829 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3830 break; 3831 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3832 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3833 break; 3834 } 3835 } 3836 3837 static int 3838 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3839 struct mlxsw_sp_nexthop_group *nh_grp) 3840 { 3841 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3842 u16 ecmp_size, old_ecmp_size; 3843 struct mlxsw_sp_nexthop *nh; 3844 bool offload_change = false; 3845 u32 adj_index; 3846 bool old_adj_index_valid; 3847 u32 old_adj_index; 3848 int i, err2, err; 3849 3850 if (!nhgi->gateway) 3851 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3852 3853 for (i = 0; i < nhgi->count; i++) { 3854 nh = &nhgi->nexthops[i]; 3855 3856 if (nh->should_offload != nh->offloaded) { 3857 offload_change = true; 3858 if (nh->should_offload) 3859 nh->update = 1; 3860 } 3861 } 3862 if (!offload_change) { 3863 /* Nothing was added or removed, so no need to reallocate. Just 3864 * update MAC on existing adjacency indexes. 3865 */ 3866 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3867 if (err) { 3868 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3869 goto set_trap; 3870 } 3871 /* Flags of individual nexthop buckets might need to be 3872 * updated. 3873 */ 3874 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3875 return 0; 3876 } 3877 mlxsw_sp_nexthop_group_normalize(nhgi); 3878 if (!nhgi->sum_norm_weight) { 3879 /* No neigh of this group is connected so we just set 3880 * the trap and let everthing flow through kernel. 3881 */ 3882 err = 0; 3883 goto set_trap; 3884 } 3885 3886 ecmp_size = nhgi->sum_norm_weight; 3887 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3888 if (err) 3889 /* No valid allocation size available. */ 3890 goto set_trap; 3891 3892 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3893 ecmp_size, &adj_index); 3894 if (err) { 3895 /* We ran out of KVD linear space, just set the 3896 * trap and let everything flow through kernel. 3897 */ 3898 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3899 goto set_trap; 3900 } 3901 old_adj_index_valid = nhgi->adj_index_valid; 3902 old_adj_index = nhgi->adj_index; 3903 old_ecmp_size = nhgi->ecmp_size; 3904 nhgi->adj_index_valid = 1; 3905 nhgi->adj_index = adj_index; 3906 nhgi->ecmp_size = ecmp_size; 3907 mlxsw_sp_nexthop_group_rebalance(nhgi); 3908 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3909 if (err) { 3910 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3911 goto set_trap; 3912 } 3913 3914 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3915 3916 if (!old_adj_index_valid) { 3917 /* The trap was set for fib entries, so we have to call 3918 * fib entry update to unset it and use adjacency index. 3919 */ 3920 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3921 if (err) { 3922 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3923 goto set_trap; 3924 } 3925 return 0; 3926 } 3927 3928 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3929 old_adj_index, old_ecmp_size); 3930 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3931 old_ecmp_size, old_adj_index); 3932 if (err) { 3933 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3934 goto set_trap; 3935 } 3936 3937 return 0; 3938 3939 set_trap: 3940 old_adj_index_valid = nhgi->adj_index_valid; 3941 nhgi->adj_index_valid = 0; 3942 for (i = 0; i < nhgi->count; i++) { 3943 nh = &nhgi->nexthops[i]; 3944 nh->offloaded = 0; 3945 } 3946 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3947 if (err2) 3948 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3949 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3950 if (old_adj_index_valid) 3951 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3952 nhgi->ecmp_size, nhgi->adj_index); 3953 return err; 3954 } 3955 3956 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3957 bool removing) 3958 { 3959 if (!removing) { 3960 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3961 nh->should_offload = 1; 3962 } else if (nh->nhgi->is_resilient) { 3963 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 3964 nh->should_offload = 1; 3965 } else { 3966 nh->should_offload = 0; 3967 } 3968 nh->update = 1; 3969 } 3970 3971 static int 3972 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 3973 struct mlxsw_sp_neigh_entry *neigh_entry) 3974 { 3975 struct neighbour *n, *old_n = neigh_entry->key.n; 3976 struct mlxsw_sp_nexthop *nh; 3977 bool entry_connected; 3978 u8 nud_state, dead; 3979 int err; 3980 3981 nh = list_first_entry(&neigh_entry->nexthop_list, 3982 struct mlxsw_sp_nexthop, neigh_list_node); 3983 3984 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3985 if (!n) { 3986 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 3987 if (IS_ERR(n)) 3988 return PTR_ERR(n); 3989 neigh_event_send(n, NULL); 3990 } 3991 3992 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 3993 neigh_entry->key.n = n; 3994 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 3995 if (err) 3996 goto err_neigh_entry_insert; 3997 3998 read_lock_bh(&n->lock); 3999 nud_state = n->nud_state; 4000 dead = n->dead; 4001 read_unlock_bh(&n->lock); 4002 entry_connected = nud_state & NUD_VALID && !dead; 4003 4004 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4005 neigh_list_node) { 4006 neigh_release(old_n); 4007 neigh_clone(n); 4008 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4009 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4010 } 4011 4012 neigh_release(n); 4013 4014 return 0; 4015 4016 err_neigh_entry_insert: 4017 neigh_entry->key.n = old_n; 4018 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4019 neigh_release(n); 4020 return err; 4021 } 4022 4023 static void 4024 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4025 struct mlxsw_sp_neigh_entry *neigh_entry, 4026 bool removing, bool dead) 4027 { 4028 struct mlxsw_sp_nexthop *nh; 4029 4030 if (list_empty(&neigh_entry->nexthop_list)) 4031 return; 4032 4033 if (dead) { 4034 int err; 4035 4036 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4037 neigh_entry); 4038 if (err) 4039 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4040 return; 4041 } 4042 4043 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4044 neigh_list_node) { 4045 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4046 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4047 } 4048 } 4049 4050 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 4051 struct mlxsw_sp_rif *rif) 4052 { 4053 if (nh->rif) 4054 return; 4055 4056 nh->rif = rif; 4057 list_add(&nh->rif_list_node, &rif->nexthop_list); 4058 } 4059 4060 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4061 { 4062 if (!nh->rif) 4063 return; 4064 4065 list_del(&nh->rif_list_node); 4066 nh->rif = NULL; 4067 } 4068 4069 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4070 struct mlxsw_sp_nexthop *nh) 4071 { 4072 struct mlxsw_sp_neigh_entry *neigh_entry; 4073 struct neighbour *n; 4074 u8 nud_state, dead; 4075 int err; 4076 4077 if (!nh->nhgi->gateway || nh->neigh_entry) 4078 return 0; 4079 4080 /* Take a reference of neigh here ensuring that neigh would 4081 * not be destructed before the nexthop entry is finished. 4082 * The reference is taken either in neigh_lookup() or 4083 * in neigh_create() in case n is not found. 4084 */ 4085 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4086 if (!n) { 4087 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4088 if (IS_ERR(n)) 4089 return PTR_ERR(n); 4090 neigh_event_send(n, NULL); 4091 } 4092 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4093 if (!neigh_entry) { 4094 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4095 if (IS_ERR(neigh_entry)) { 4096 err = -EINVAL; 4097 goto err_neigh_entry_create; 4098 } 4099 } 4100 4101 /* If that is the first nexthop connected to that neigh, add to 4102 * nexthop_neighs_list 4103 */ 4104 if (list_empty(&neigh_entry->nexthop_list)) 4105 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4106 &mlxsw_sp->router->nexthop_neighs_list); 4107 4108 nh->neigh_entry = neigh_entry; 4109 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4110 read_lock_bh(&n->lock); 4111 nud_state = n->nud_state; 4112 dead = n->dead; 4113 read_unlock_bh(&n->lock); 4114 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4115 4116 return 0; 4117 4118 err_neigh_entry_create: 4119 neigh_release(n); 4120 return err; 4121 } 4122 4123 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4124 struct mlxsw_sp_nexthop *nh) 4125 { 4126 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4127 struct neighbour *n; 4128 4129 if (!neigh_entry) 4130 return; 4131 n = neigh_entry->key.n; 4132 4133 __mlxsw_sp_nexthop_neigh_update(nh, true); 4134 list_del(&nh->neigh_list_node); 4135 nh->neigh_entry = NULL; 4136 4137 /* If that is the last nexthop connected to that neigh, remove from 4138 * nexthop_neighs_list 4139 */ 4140 if (list_empty(&neigh_entry->nexthop_list)) 4141 list_del(&neigh_entry->nexthop_neighs_list_node); 4142 4143 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4144 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4145 4146 neigh_release(n); 4147 } 4148 4149 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4150 { 4151 struct net_device *ul_dev; 4152 bool is_up; 4153 4154 rcu_read_lock(); 4155 ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4156 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4157 rcu_read_unlock(); 4158 4159 return is_up; 4160 } 4161 4162 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4163 struct mlxsw_sp_nexthop *nh, 4164 struct mlxsw_sp_ipip_entry *ipip_entry) 4165 { 4166 bool removing; 4167 4168 if (!nh->nhgi->gateway || nh->ipip_entry) 4169 return; 4170 4171 nh->ipip_entry = ipip_entry; 4172 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4173 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4174 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4175 } 4176 4177 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4178 struct mlxsw_sp_nexthop *nh) 4179 { 4180 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4181 4182 if (!ipip_entry) 4183 return; 4184 4185 __mlxsw_sp_nexthop_neigh_update(nh, true); 4186 nh->ipip_entry = NULL; 4187 } 4188 4189 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4190 const struct fib_nh *fib_nh, 4191 enum mlxsw_sp_ipip_type *p_ipipt) 4192 { 4193 struct net_device *dev = fib_nh->fib_nh_dev; 4194 4195 return dev && 4196 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4197 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4198 } 4199 4200 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4201 struct mlxsw_sp_nexthop *nh, 4202 const struct net_device *dev) 4203 { 4204 const struct mlxsw_sp_ipip_ops *ipip_ops; 4205 struct mlxsw_sp_ipip_entry *ipip_entry; 4206 struct mlxsw_sp_rif *rif; 4207 int err; 4208 4209 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4210 if (ipip_entry) { 4211 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4212 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4213 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4214 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4215 return 0; 4216 } 4217 } 4218 4219 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4220 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4221 if (!rif) 4222 return 0; 4223 4224 mlxsw_sp_nexthop_rif_init(nh, rif); 4225 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4226 if (err) 4227 goto err_neigh_init; 4228 4229 return 0; 4230 4231 err_neigh_init: 4232 mlxsw_sp_nexthop_rif_fini(nh); 4233 return err; 4234 } 4235 4236 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4237 struct mlxsw_sp_nexthop *nh) 4238 { 4239 switch (nh->type) { 4240 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4241 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4242 mlxsw_sp_nexthop_rif_fini(nh); 4243 break; 4244 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4245 mlxsw_sp_nexthop_rif_fini(nh); 4246 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4247 break; 4248 } 4249 } 4250 4251 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4252 struct mlxsw_sp_nexthop_group *nh_grp, 4253 struct mlxsw_sp_nexthop *nh, 4254 struct fib_nh *fib_nh) 4255 { 4256 struct net_device *dev = fib_nh->fib_nh_dev; 4257 struct in_device *in_dev; 4258 int err; 4259 4260 nh->nhgi = nh_grp->nhgi; 4261 nh->key.fib_nh = fib_nh; 4262 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4263 nh->nh_weight = fib_nh->fib_nh_weight; 4264 #else 4265 nh->nh_weight = 1; 4266 #endif 4267 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4268 nh->neigh_tbl = &arp_tbl; 4269 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4270 if (err) 4271 return err; 4272 4273 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4274 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4275 4276 if (!dev) 4277 return 0; 4278 nh->ifindex = dev->ifindex; 4279 4280 rcu_read_lock(); 4281 in_dev = __in_dev_get_rcu(dev); 4282 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4283 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4284 rcu_read_unlock(); 4285 return 0; 4286 } 4287 rcu_read_unlock(); 4288 4289 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4290 if (err) 4291 goto err_nexthop_neigh_init; 4292 4293 return 0; 4294 4295 err_nexthop_neigh_init: 4296 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4297 return err; 4298 } 4299 4300 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4301 struct mlxsw_sp_nexthop *nh) 4302 { 4303 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4304 list_del(&nh->router_list_node); 4305 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4306 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4307 } 4308 4309 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4310 unsigned long event, struct fib_nh *fib_nh) 4311 { 4312 struct mlxsw_sp_nexthop_key key; 4313 struct mlxsw_sp_nexthop *nh; 4314 4315 if (mlxsw_sp->router->aborted) 4316 return; 4317 4318 key.fib_nh = fib_nh; 4319 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4320 if (!nh) 4321 return; 4322 4323 switch (event) { 4324 case FIB_EVENT_NH_ADD: 4325 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4326 break; 4327 case FIB_EVENT_NH_DEL: 4328 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4329 break; 4330 } 4331 4332 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4333 } 4334 4335 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4336 struct mlxsw_sp_rif *rif) 4337 { 4338 struct mlxsw_sp_nexthop *nh; 4339 bool removing; 4340 4341 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4342 switch (nh->type) { 4343 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4344 removing = false; 4345 break; 4346 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4347 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4348 break; 4349 default: 4350 WARN_ON(1); 4351 continue; 4352 } 4353 4354 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4355 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4356 } 4357 } 4358 4359 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4360 struct mlxsw_sp_rif *old_rif, 4361 struct mlxsw_sp_rif *new_rif) 4362 { 4363 struct mlxsw_sp_nexthop *nh; 4364 4365 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4366 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4367 nh->rif = new_rif; 4368 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4369 } 4370 4371 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4372 struct mlxsw_sp_rif *rif) 4373 { 4374 struct mlxsw_sp_nexthop *nh, *tmp; 4375 4376 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4377 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4378 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4379 } 4380 } 4381 4382 static void 4383 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4384 const struct mlxsw_sp_nexthop_group *nh_grp, 4385 unsigned long *activity) 4386 { 4387 char *ratrad_pl; 4388 int i, err; 4389 4390 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4391 if (!ratrad_pl) 4392 return; 4393 4394 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4395 nh_grp->nhgi->count); 4396 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4397 if (err) 4398 goto out; 4399 4400 for (i = 0; i < nh_grp->nhgi->count; i++) { 4401 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4402 continue; 4403 bitmap_set(activity, i, 1); 4404 } 4405 4406 out: 4407 kfree(ratrad_pl); 4408 } 4409 4410 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4411 4412 static void 4413 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4414 const struct mlxsw_sp_nexthop_group *nh_grp) 4415 { 4416 unsigned long *activity; 4417 4418 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4419 if (!activity) 4420 return; 4421 4422 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4423 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4424 nh_grp->nhgi->count, activity); 4425 4426 bitmap_free(activity); 4427 } 4428 4429 static void 4430 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4431 { 4432 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4433 4434 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4435 msecs_to_jiffies(interval)); 4436 } 4437 4438 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4439 { 4440 struct mlxsw_sp_nexthop_group_info *nhgi; 4441 struct mlxsw_sp_router *router; 4442 bool reschedule = false; 4443 4444 router = container_of(work, struct mlxsw_sp_router, 4445 nh_grp_activity_dw.work); 4446 4447 mutex_lock(&router->lock); 4448 4449 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4450 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4451 reschedule = true; 4452 } 4453 4454 mutex_unlock(&router->lock); 4455 4456 if (!reschedule) 4457 return; 4458 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4459 } 4460 4461 static int 4462 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4463 const struct nh_notifier_single_info *nh, 4464 struct netlink_ext_ack *extack) 4465 { 4466 int err = -EINVAL; 4467 4468 if (nh->is_fdb) 4469 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4470 else if (nh->has_encap) 4471 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4472 else 4473 err = 0; 4474 4475 return err; 4476 } 4477 4478 static int 4479 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4480 const struct nh_notifier_single_info *nh, 4481 struct netlink_ext_ack *extack) 4482 { 4483 int err; 4484 4485 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4486 if (err) 4487 return err; 4488 4489 /* Device only nexthops with an IPIP device are programmed as 4490 * encapsulating adjacency entries. 4491 */ 4492 if (!nh->gw_family && !nh->is_reject && 4493 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4494 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4495 return -EINVAL; 4496 } 4497 4498 return 0; 4499 } 4500 4501 static int 4502 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4503 const struct nh_notifier_grp_info *nh_grp, 4504 struct netlink_ext_ack *extack) 4505 { 4506 int i; 4507 4508 if (nh_grp->is_fdb) { 4509 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4510 return -EINVAL; 4511 } 4512 4513 for (i = 0; i < nh_grp->num_nh; i++) { 4514 const struct nh_notifier_single_info *nh; 4515 int err; 4516 4517 nh = &nh_grp->nh_entries[i].nh; 4518 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4519 extack); 4520 if (err) 4521 return err; 4522 } 4523 4524 return 0; 4525 } 4526 4527 static int 4528 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4529 const struct nh_notifier_res_table_info *nh_res_table, 4530 struct netlink_ext_ack *extack) 4531 { 4532 unsigned int alloc_size; 4533 bool valid_size = false; 4534 int err, i; 4535 4536 if (nh_res_table->num_nh_buckets < 32) { 4537 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4538 return -EINVAL; 4539 } 4540 4541 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4542 const struct mlxsw_sp_adj_grp_size_range *size_range; 4543 4544 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4545 4546 if (nh_res_table->num_nh_buckets >= size_range->start && 4547 nh_res_table->num_nh_buckets <= size_range->end) { 4548 valid_size = true; 4549 break; 4550 } 4551 } 4552 4553 if (!valid_size) { 4554 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4555 return -EINVAL; 4556 } 4557 4558 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4559 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4560 nh_res_table->num_nh_buckets, 4561 &alloc_size); 4562 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4563 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4564 return -EINVAL; 4565 } 4566 4567 return 0; 4568 } 4569 4570 static int 4571 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4572 const struct nh_notifier_res_table_info *nh_res_table, 4573 struct netlink_ext_ack *extack) 4574 { 4575 int err; 4576 u16 i; 4577 4578 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4579 nh_res_table, 4580 extack); 4581 if (err) 4582 return err; 4583 4584 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4585 const struct nh_notifier_single_info *nh; 4586 int err; 4587 4588 nh = &nh_res_table->nhs[i]; 4589 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4590 extack); 4591 if (err) 4592 return err; 4593 } 4594 4595 return 0; 4596 } 4597 4598 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4599 unsigned long event, 4600 struct nh_notifier_info *info) 4601 { 4602 struct nh_notifier_single_info *nh; 4603 4604 if (event != NEXTHOP_EVENT_REPLACE && 4605 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4606 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4607 return 0; 4608 4609 switch (info->type) { 4610 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4611 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4612 info->extack); 4613 case NH_NOTIFIER_INFO_TYPE_GRP: 4614 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4615 info->nh_grp, 4616 info->extack); 4617 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4618 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4619 info->nh_res_table, 4620 info->extack); 4621 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4622 nh = &info->nh_res_bucket->new_nh; 4623 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4624 info->extack); 4625 default: 4626 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4627 return -EOPNOTSUPP; 4628 } 4629 } 4630 4631 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4632 const struct nh_notifier_info *info) 4633 { 4634 const struct net_device *dev; 4635 4636 switch (info->type) { 4637 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4638 dev = info->nh->dev; 4639 return info->nh->gw_family || info->nh->is_reject || 4640 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4641 case NH_NOTIFIER_INFO_TYPE_GRP: 4642 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4643 /* Already validated earlier. */ 4644 return true; 4645 default: 4646 return false; 4647 } 4648 } 4649 4650 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4651 struct mlxsw_sp_nexthop *nh) 4652 { 4653 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4654 4655 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4656 nh->should_offload = 1; 4657 /* While nexthops that discard packets do not forward packets 4658 * via an egress RIF, they still need to be programmed using a 4659 * valid RIF, so use the loopback RIF created during init. 4660 */ 4661 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4662 } 4663 4664 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4665 struct mlxsw_sp_nexthop *nh) 4666 { 4667 nh->rif = NULL; 4668 nh->should_offload = 0; 4669 } 4670 4671 static int 4672 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4673 struct mlxsw_sp_nexthop_group *nh_grp, 4674 struct mlxsw_sp_nexthop *nh, 4675 struct nh_notifier_single_info *nh_obj, int weight) 4676 { 4677 struct net_device *dev = nh_obj->dev; 4678 int err; 4679 4680 nh->nhgi = nh_grp->nhgi; 4681 nh->nh_weight = weight; 4682 4683 switch (nh_obj->gw_family) { 4684 case AF_INET: 4685 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4686 nh->neigh_tbl = &arp_tbl; 4687 break; 4688 case AF_INET6: 4689 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4690 #if IS_ENABLED(CONFIG_IPV6) 4691 nh->neigh_tbl = &nd_tbl; 4692 #endif 4693 break; 4694 } 4695 4696 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4697 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4698 nh->ifindex = dev->ifindex; 4699 4700 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4701 if (err) 4702 goto err_type_init; 4703 4704 if (nh_obj->is_reject) 4705 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4706 4707 /* In a resilient nexthop group, all the nexthops must be written to 4708 * the adjacency table. Even if they do not have a valid neighbour or 4709 * RIF. 4710 */ 4711 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4712 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4713 nh->should_offload = 1; 4714 } 4715 4716 return 0; 4717 4718 err_type_init: 4719 list_del(&nh->router_list_node); 4720 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4721 return err; 4722 } 4723 4724 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4725 struct mlxsw_sp_nexthop *nh) 4726 { 4727 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4728 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4729 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4730 list_del(&nh->router_list_node); 4731 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4732 nh->should_offload = 0; 4733 } 4734 4735 static int 4736 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4737 struct mlxsw_sp_nexthop_group *nh_grp, 4738 struct nh_notifier_info *info) 4739 { 4740 struct mlxsw_sp_nexthop_group_info *nhgi; 4741 struct mlxsw_sp_nexthop *nh; 4742 bool is_resilient = false; 4743 unsigned int nhs; 4744 int err, i; 4745 4746 switch (info->type) { 4747 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4748 nhs = 1; 4749 break; 4750 case NH_NOTIFIER_INFO_TYPE_GRP: 4751 nhs = info->nh_grp->num_nh; 4752 break; 4753 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4754 nhs = info->nh_res_table->num_nh_buckets; 4755 is_resilient = true; 4756 break; 4757 default: 4758 return -EINVAL; 4759 } 4760 4761 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4762 if (!nhgi) 4763 return -ENOMEM; 4764 nh_grp->nhgi = nhgi; 4765 nhgi->nh_grp = nh_grp; 4766 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4767 nhgi->is_resilient = is_resilient; 4768 nhgi->count = nhs; 4769 for (i = 0; i < nhgi->count; i++) { 4770 struct nh_notifier_single_info *nh_obj; 4771 int weight; 4772 4773 nh = &nhgi->nexthops[i]; 4774 switch (info->type) { 4775 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4776 nh_obj = info->nh; 4777 weight = 1; 4778 break; 4779 case NH_NOTIFIER_INFO_TYPE_GRP: 4780 nh_obj = &info->nh_grp->nh_entries[i].nh; 4781 weight = info->nh_grp->nh_entries[i].weight; 4782 break; 4783 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4784 nh_obj = &info->nh_res_table->nhs[i]; 4785 weight = 1; 4786 break; 4787 default: 4788 err = -EINVAL; 4789 goto err_nexthop_obj_init; 4790 } 4791 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4792 weight); 4793 if (err) 4794 goto err_nexthop_obj_init; 4795 } 4796 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4797 if (err) { 4798 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4799 goto err_group_refresh; 4800 } 4801 4802 /* Add resilient nexthop groups to a list so that the activity of their 4803 * nexthop buckets will be periodically queried and cleared. 4804 */ 4805 if (nhgi->is_resilient) { 4806 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4807 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 4808 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 4809 } 4810 4811 return 0; 4812 4813 err_group_refresh: 4814 i = nhgi->count; 4815 err_nexthop_obj_init: 4816 for (i--; i >= 0; i--) { 4817 nh = &nhgi->nexthops[i]; 4818 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4819 } 4820 kfree(nhgi); 4821 return err; 4822 } 4823 4824 static void 4825 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4826 struct mlxsw_sp_nexthop_group *nh_grp) 4827 { 4828 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4829 struct mlxsw_sp_router *router = mlxsw_sp->router; 4830 int i; 4831 4832 if (nhgi->is_resilient) { 4833 list_del(&nhgi->list); 4834 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4835 cancel_delayed_work(&router->nh_grp_activity_dw); 4836 } 4837 4838 for (i = nhgi->count - 1; i >= 0; i--) { 4839 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4840 4841 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4842 } 4843 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4844 WARN_ON_ONCE(nhgi->adj_index_valid); 4845 kfree(nhgi); 4846 } 4847 4848 static struct mlxsw_sp_nexthop_group * 4849 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4850 struct nh_notifier_info *info) 4851 { 4852 struct mlxsw_sp_nexthop_group *nh_grp; 4853 int err; 4854 4855 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4856 if (!nh_grp) 4857 return ERR_PTR(-ENOMEM); 4858 INIT_LIST_HEAD(&nh_grp->vr_list); 4859 err = rhashtable_init(&nh_grp->vr_ht, 4860 &mlxsw_sp_nexthop_group_vr_ht_params); 4861 if (err) 4862 goto err_nexthop_group_vr_ht_init; 4863 INIT_LIST_HEAD(&nh_grp->fib_list); 4864 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4865 nh_grp->obj.id = info->id; 4866 4867 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4868 if (err) 4869 goto err_nexthop_group_info_init; 4870 4871 nh_grp->can_destroy = false; 4872 4873 return nh_grp; 4874 4875 err_nexthop_group_info_init: 4876 rhashtable_destroy(&nh_grp->vr_ht); 4877 err_nexthop_group_vr_ht_init: 4878 kfree(nh_grp); 4879 return ERR_PTR(err); 4880 } 4881 4882 static void 4883 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4884 struct mlxsw_sp_nexthop_group *nh_grp) 4885 { 4886 if (!nh_grp->can_destroy) 4887 return; 4888 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4889 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4890 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4891 rhashtable_destroy(&nh_grp->vr_ht); 4892 kfree(nh_grp); 4893 } 4894 4895 static struct mlxsw_sp_nexthop_group * 4896 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4897 { 4898 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4899 4900 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4901 cmp_arg.id = id; 4902 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4903 &cmp_arg, 4904 mlxsw_sp_nexthop_group_ht_params); 4905 } 4906 4907 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 4908 struct mlxsw_sp_nexthop_group *nh_grp) 4909 { 4910 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 4911 } 4912 4913 static int 4914 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 4915 struct mlxsw_sp_nexthop_group *nh_grp, 4916 struct mlxsw_sp_nexthop_group *old_nh_grp, 4917 struct netlink_ext_ack *extack) 4918 { 4919 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 4920 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 4921 int err; 4922 4923 old_nh_grp->nhgi = new_nhgi; 4924 new_nhgi->nh_grp = old_nh_grp; 4925 nh_grp->nhgi = old_nhgi; 4926 old_nhgi->nh_grp = nh_grp; 4927 4928 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4929 /* Both the old adjacency index and the new one are valid. 4930 * Routes are currently using the old one. Tell the device to 4931 * replace the old adjacency index with the new one. 4932 */ 4933 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 4934 old_nhgi->adj_index, 4935 old_nhgi->ecmp_size); 4936 if (err) { 4937 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 4938 goto err_out; 4939 } 4940 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 4941 /* The old adjacency index is valid, while the new one is not. 4942 * Iterate over all the routes using the group and change them 4943 * to trap packets to the CPU. 4944 */ 4945 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4946 if (err) { 4947 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 4948 goto err_out; 4949 } 4950 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 4951 /* The old adjacency index is invalid, while the new one is. 4952 * Iterate over all the routes using the group and change them 4953 * to forward packets using the new valid index. 4954 */ 4955 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 4956 if (err) { 4957 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 4958 goto err_out; 4959 } 4960 } 4961 4962 /* Make sure the flags are set / cleared based on the new nexthop group 4963 * information. 4964 */ 4965 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 4966 4967 /* At this point 'nh_grp' is just a shell that is not used by anyone 4968 * and its nexthop group info is the old info that was just replaced 4969 * with the new one. Remove it. 4970 */ 4971 nh_grp->can_destroy = true; 4972 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 4973 4974 return 0; 4975 4976 err_out: 4977 old_nhgi->nh_grp = old_nh_grp; 4978 nh_grp->nhgi = new_nhgi; 4979 new_nhgi->nh_grp = nh_grp; 4980 old_nh_grp->nhgi = old_nhgi; 4981 return err; 4982 } 4983 4984 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 4985 struct nh_notifier_info *info) 4986 { 4987 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 4988 struct netlink_ext_ack *extack = info->extack; 4989 int err; 4990 4991 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 4992 if (IS_ERR(nh_grp)) 4993 return PTR_ERR(nh_grp); 4994 4995 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 4996 if (!old_nh_grp) 4997 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 4998 else 4999 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5000 old_nh_grp, extack); 5001 5002 if (err) { 5003 nh_grp->can_destroy = true; 5004 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5005 } 5006 5007 return err; 5008 } 5009 5010 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5011 struct nh_notifier_info *info) 5012 { 5013 struct mlxsw_sp_nexthop_group *nh_grp; 5014 5015 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5016 if (!nh_grp) 5017 return; 5018 5019 nh_grp->can_destroy = true; 5020 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5021 5022 /* If the group still has routes using it, then defer the delete 5023 * operation until the last route using it is deleted. 5024 */ 5025 if (!list_empty(&nh_grp->fib_list)) 5026 return; 5027 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5028 } 5029 5030 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5031 u32 adj_index, char *ratr_pl) 5032 { 5033 MLXSW_REG_ZERO(ratr, ratr_pl); 5034 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5035 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5036 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5037 5038 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5039 } 5040 5041 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5042 { 5043 /* Clear the opcode and activity on both the old and new payload as 5044 * they are irrelevant for the comparison. 5045 */ 5046 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5047 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5048 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5049 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5050 5051 /* If the contents of the adjacency entry are consistent with the 5052 * replacement request, then replacement was successful. 5053 */ 5054 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5055 return 0; 5056 5057 return -EINVAL; 5058 } 5059 5060 static int 5061 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5062 struct mlxsw_sp_nexthop *nh, 5063 struct nh_notifier_info *info) 5064 { 5065 u16 bucket_index = info->nh_res_bucket->bucket_index; 5066 struct netlink_ext_ack *extack = info->extack; 5067 bool force = info->nh_res_bucket->force; 5068 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5069 char ratr_pl[MLXSW_REG_RATR_LEN]; 5070 u32 adj_index; 5071 int err; 5072 5073 /* No point in trying an atomic replacement if the idle timer interval 5074 * is smaller than the interval in which we query and clear activity. 5075 */ 5076 if (!force && info->nh_res_bucket->idle_timer_ms < 5077 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5078 force = true; 5079 5080 adj_index = nh->nhgi->adj_index + bucket_index; 5081 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5082 if (err) { 5083 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5084 return err; 5085 } 5086 5087 if (!force) { 5088 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5089 ratr_pl_new); 5090 if (err) { 5091 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5092 return err; 5093 } 5094 5095 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5096 if (err) { 5097 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5098 return err; 5099 } 5100 } 5101 5102 nh->update = 0; 5103 nh->offloaded = 1; 5104 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5105 5106 return 0; 5107 } 5108 5109 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5110 struct nh_notifier_info *info) 5111 { 5112 u16 bucket_index = info->nh_res_bucket->bucket_index; 5113 struct netlink_ext_ack *extack = info->extack; 5114 struct mlxsw_sp_nexthop_group_info *nhgi; 5115 struct nh_notifier_single_info *nh_obj; 5116 struct mlxsw_sp_nexthop_group *nh_grp; 5117 struct mlxsw_sp_nexthop *nh; 5118 int err; 5119 5120 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5121 if (!nh_grp) { 5122 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5123 return -EINVAL; 5124 } 5125 5126 nhgi = nh_grp->nhgi; 5127 5128 if (bucket_index >= nhgi->count) { 5129 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5130 return -EINVAL; 5131 } 5132 5133 nh = &nhgi->nexthops[bucket_index]; 5134 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5135 5136 nh_obj = &info->nh_res_bucket->new_nh; 5137 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5138 if (err) { 5139 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5140 goto err_nexthop_obj_init; 5141 } 5142 5143 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5144 if (err) 5145 goto err_nexthop_obj_bucket_adj_update; 5146 5147 return 0; 5148 5149 err_nexthop_obj_bucket_adj_update: 5150 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5151 err_nexthop_obj_init: 5152 nh_obj = &info->nh_res_bucket->old_nh; 5153 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5154 /* The old adjacency entry was not overwritten */ 5155 nh->update = 0; 5156 nh->offloaded = 1; 5157 return err; 5158 } 5159 5160 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5161 unsigned long event, void *ptr) 5162 { 5163 struct nh_notifier_info *info = ptr; 5164 struct mlxsw_sp_router *router; 5165 int err = 0; 5166 5167 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5168 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5169 if (err) 5170 goto out; 5171 5172 mutex_lock(&router->lock); 5173 5174 switch (event) { 5175 case NEXTHOP_EVENT_REPLACE: 5176 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5177 break; 5178 case NEXTHOP_EVENT_DEL: 5179 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5180 break; 5181 case NEXTHOP_EVENT_BUCKET_REPLACE: 5182 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5183 info); 5184 break; 5185 default: 5186 break; 5187 } 5188 5189 mutex_unlock(&router->lock); 5190 5191 out: 5192 return notifier_from_errno(err); 5193 } 5194 5195 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5196 struct fib_info *fi) 5197 { 5198 const struct fib_nh *nh = fib_info_nh(fi, 0); 5199 5200 return nh->fib_nh_scope == RT_SCOPE_LINK || 5201 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5202 } 5203 5204 static int 5205 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5206 struct mlxsw_sp_nexthop_group *nh_grp) 5207 { 5208 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5209 struct mlxsw_sp_nexthop_group_info *nhgi; 5210 struct mlxsw_sp_nexthop *nh; 5211 int err, i; 5212 5213 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5214 if (!nhgi) 5215 return -ENOMEM; 5216 nh_grp->nhgi = nhgi; 5217 nhgi->nh_grp = nh_grp; 5218 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5219 nhgi->count = nhs; 5220 for (i = 0; i < nhgi->count; i++) { 5221 struct fib_nh *fib_nh; 5222 5223 nh = &nhgi->nexthops[i]; 5224 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5225 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5226 if (err) 5227 goto err_nexthop4_init; 5228 } 5229 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5230 if (err) 5231 goto err_group_refresh; 5232 5233 return 0; 5234 5235 err_group_refresh: 5236 i = nhgi->count; 5237 err_nexthop4_init: 5238 for (i--; i >= 0; i--) { 5239 nh = &nhgi->nexthops[i]; 5240 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5241 } 5242 kfree(nhgi); 5243 return err; 5244 } 5245 5246 static void 5247 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5248 struct mlxsw_sp_nexthop_group *nh_grp) 5249 { 5250 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5251 int i; 5252 5253 for (i = nhgi->count - 1; i >= 0; i--) { 5254 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5255 5256 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5257 } 5258 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5259 WARN_ON_ONCE(nhgi->adj_index_valid); 5260 kfree(nhgi); 5261 } 5262 5263 static struct mlxsw_sp_nexthop_group * 5264 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5265 { 5266 struct mlxsw_sp_nexthop_group *nh_grp; 5267 int err; 5268 5269 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5270 if (!nh_grp) 5271 return ERR_PTR(-ENOMEM); 5272 INIT_LIST_HEAD(&nh_grp->vr_list); 5273 err = rhashtable_init(&nh_grp->vr_ht, 5274 &mlxsw_sp_nexthop_group_vr_ht_params); 5275 if (err) 5276 goto err_nexthop_group_vr_ht_init; 5277 INIT_LIST_HEAD(&nh_grp->fib_list); 5278 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5279 nh_grp->ipv4.fi = fi; 5280 fib_info_hold(fi); 5281 5282 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5283 if (err) 5284 goto err_nexthop_group_info_init; 5285 5286 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5287 if (err) 5288 goto err_nexthop_group_insert; 5289 5290 nh_grp->can_destroy = true; 5291 5292 return nh_grp; 5293 5294 err_nexthop_group_insert: 5295 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5296 err_nexthop_group_info_init: 5297 fib_info_put(fi); 5298 rhashtable_destroy(&nh_grp->vr_ht); 5299 err_nexthop_group_vr_ht_init: 5300 kfree(nh_grp); 5301 return ERR_PTR(err); 5302 } 5303 5304 static void 5305 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5306 struct mlxsw_sp_nexthop_group *nh_grp) 5307 { 5308 if (!nh_grp->can_destroy) 5309 return; 5310 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5311 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5312 fib_info_put(nh_grp->ipv4.fi); 5313 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5314 rhashtable_destroy(&nh_grp->vr_ht); 5315 kfree(nh_grp); 5316 } 5317 5318 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5319 struct mlxsw_sp_fib_entry *fib_entry, 5320 struct fib_info *fi) 5321 { 5322 struct mlxsw_sp_nexthop_group *nh_grp; 5323 5324 if (fi->nh) { 5325 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5326 fi->nh->id); 5327 if (WARN_ON_ONCE(!nh_grp)) 5328 return -EINVAL; 5329 goto out; 5330 } 5331 5332 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5333 if (!nh_grp) { 5334 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5335 if (IS_ERR(nh_grp)) 5336 return PTR_ERR(nh_grp); 5337 } 5338 out: 5339 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5340 fib_entry->nh_group = nh_grp; 5341 return 0; 5342 } 5343 5344 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5345 struct mlxsw_sp_fib_entry *fib_entry) 5346 { 5347 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5348 5349 list_del(&fib_entry->nexthop_group_node); 5350 if (!list_empty(&nh_grp->fib_list)) 5351 return; 5352 5353 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5354 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5355 return; 5356 } 5357 5358 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5359 } 5360 5361 static bool 5362 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5363 { 5364 struct mlxsw_sp_fib4_entry *fib4_entry; 5365 5366 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5367 common); 5368 return !fib4_entry->tos; 5369 } 5370 5371 static bool 5372 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5373 { 5374 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5375 5376 switch (fib_entry->fib_node->fib->proto) { 5377 case MLXSW_SP_L3_PROTO_IPV4: 5378 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5379 return false; 5380 break; 5381 case MLXSW_SP_L3_PROTO_IPV6: 5382 break; 5383 } 5384 5385 switch (fib_entry->type) { 5386 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5387 return !!nh_group->nhgi->adj_index_valid; 5388 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5389 return !!nh_group->nhgi->nh_rif; 5390 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5391 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5392 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5393 return true; 5394 default: 5395 return false; 5396 } 5397 } 5398 5399 static struct mlxsw_sp_nexthop * 5400 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5401 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5402 { 5403 int i; 5404 5405 for (i = 0; i < nh_grp->nhgi->count; i++) { 5406 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5407 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5408 5409 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5410 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5411 &rt->fib6_nh->fib_nh_gw6)) 5412 return nh; 5413 continue; 5414 } 5415 5416 return NULL; 5417 } 5418 5419 static void 5420 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5421 struct fib_entry_notifier_info *fen_info) 5422 { 5423 u32 *p_dst = (u32 *) &fen_info->dst; 5424 struct fib_rt_info fri; 5425 5426 fri.fi = fen_info->fi; 5427 fri.tb_id = fen_info->tb_id; 5428 fri.dst = cpu_to_be32(*p_dst); 5429 fri.dst_len = fen_info->dst_len; 5430 fri.tos = fen_info->tos; 5431 fri.type = fen_info->type; 5432 fri.offload = false; 5433 fri.trap = false; 5434 fri.offload_failed = true; 5435 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5436 } 5437 5438 static void 5439 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5440 struct mlxsw_sp_fib_entry *fib_entry) 5441 { 5442 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5443 int dst_len = fib_entry->fib_node->key.prefix_len; 5444 struct mlxsw_sp_fib4_entry *fib4_entry; 5445 struct fib_rt_info fri; 5446 bool should_offload; 5447 5448 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5449 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5450 common); 5451 fri.fi = fib4_entry->fi; 5452 fri.tb_id = fib4_entry->tb_id; 5453 fri.dst = cpu_to_be32(*p_dst); 5454 fri.dst_len = dst_len; 5455 fri.tos = fib4_entry->tos; 5456 fri.type = fib4_entry->type; 5457 fri.offload = should_offload; 5458 fri.trap = !should_offload; 5459 fri.offload_failed = false; 5460 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5461 } 5462 5463 static void 5464 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5465 struct mlxsw_sp_fib_entry *fib_entry) 5466 { 5467 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5468 int dst_len = fib_entry->fib_node->key.prefix_len; 5469 struct mlxsw_sp_fib4_entry *fib4_entry; 5470 struct fib_rt_info fri; 5471 5472 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5473 common); 5474 fri.fi = fib4_entry->fi; 5475 fri.tb_id = fib4_entry->tb_id; 5476 fri.dst = cpu_to_be32(*p_dst); 5477 fri.dst_len = dst_len; 5478 fri.tos = fib4_entry->tos; 5479 fri.type = fib4_entry->type; 5480 fri.offload = false; 5481 fri.trap = false; 5482 fri.offload_failed = false; 5483 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5484 } 5485 5486 #if IS_ENABLED(CONFIG_IPV6) 5487 static void 5488 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5489 struct fib6_info **rt_arr, 5490 unsigned int nrt6) 5491 { 5492 int i; 5493 5494 /* In IPv6 a multipath route is represented using multiple routes, so 5495 * we need to set the flags on all of them. 5496 */ 5497 for (i = 0; i < nrt6; i++) 5498 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5499 false, false, true); 5500 } 5501 #else 5502 static void 5503 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5504 struct fib6_info **rt_arr, 5505 unsigned int nrt6) 5506 { 5507 } 5508 #endif 5509 5510 #if IS_ENABLED(CONFIG_IPV6) 5511 static void 5512 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5513 struct mlxsw_sp_fib_entry *fib_entry) 5514 { 5515 struct mlxsw_sp_fib6_entry *fib6_entry; 5516 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5517 bool should_offload; 5518 5519 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5520 5521 /* In IPv6 a multipath route is represented using multiple routes, so 5522 * we need to set the flags on all of them. 5523 */ 5524 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5525 common); 5526 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5527 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5528 should_offload, !should_offload, false); 5529 } 5530 #else 5531 static void 5532 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5533 struct mlxsw_sp_fib_entry *fib_entry) 5534 { 5535 } 5536 #endif 5537 5538 #if IS_ENABLED(CONFIG_IPV6) 5539 static void 5540 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5541 struct mlxsw_sp_fib_entry *fib_entry) 5542 { 5543 struct mlxsw_sp_fib6_entry *fib6_entry; 5544 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5545 5546 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5547 common); 5548 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5549 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5550 false, false, false); 5551 } 5552 #else 5553 static void 5554 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5555 struct mlxsw_sp_fib_entry *fib_entry) 5556 { 5557 } 5558 #endif 5559 5560 static void 5561 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5562 struct mlxsw_sp_fib_entry *fib_entry) 5563 { 5564 switch (fib_entry->fib_node->fib->proto) { 5565 case MLXSW_SP_L3_PROTO_IPV4: 5566 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5567 break; 5568 case MLXSW_SP_L3_PROTO_IPV6: 5569 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5570 break; 5571 } 5572 } 5573 5574 static void 5575 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5576 struct mlxsw_sp_fib_entry *fib_entry) 5577 { 5578 switch (fib_entry->fib_node->fib->proto) { 5579 case MLXSW_SP_L3_PROTO_IPV4: 5580 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5581 break; 5582 case MLXSW_SP_L3_PROTO_IPV6: 5583 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5584 break; 5585 } 5586 } 5587 5588 static void 5589 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5590 struct mlxsw_sp_fib_entry *fib_entry, 5591 enum mlxsw_sp_fib_entry_op op) 5592 { 5593 switch (op) { 5594 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5595 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5596 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5597 break; 5598 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5599 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5600 break; 5601 default: 5602 break; 5603 } 5604 } 5605 5606 struct mlxsw_sp_fib_entry_op_ctx_basic { 5607 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5608 }; 5609 5610 static void 5611 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5612 enum mlxsw_sp_l3proto proto, 5613 enum mlxsw_sp_fib_entry_op op, 5614 u16 virtual_router, u8 prefix_len, 5615 unsigned char *addr, 5616 struct mlxsw_sp_fib_entry_priv *priv) 5617 { 5618 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5619 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5620 char *ralue_pl = op_ctx_basic->ralue_pl; 5621 enum mlxsw_reg_ralue_op ralue_op; 5622 5623 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5624 5625 switch (op) { 5626 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5627 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5628 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5629 break; 5630 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5631 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5632 break; 5633 default: 5634 WARN_ON_ONCE(1); 5635 return; 5636 } 5637 5638 switch (proto) { 5639 case MLXSW_SP_L3_PROTO_IPV4: 5640 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5641 virtual_router, prefix_len, (u32 *) addr); 5642 break; 5643 case MLXSW_SP_L3_PROTO_IPV6: 5644 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5645 virtual_router, prefix_len, addr); 5646 break; 5647 } 5648 } 5649 5650 static void 5651 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5652 enum mlxsw_reg_ralue_trap_action trap_action, 5653 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5654 { 5655 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5656 5657 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5658 trap_id, adjacency_index, ecmp_size); 5659 } 5660 5661 static void 5662 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5663 enum mlxsw_reg_ralue_trap_action trap_action, 5664 u16 trap_id, u16 local_erif) 5665 { 5666 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5667 5668 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5669 trap_id, local_erif); 5670 } 5671 5672 static void 5673 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5674 { 5675 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5676 5677 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5678 } 5679 5680 static void 5681 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5682 u32 tunnel_ptr) 5683 { 5684 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5685 5686 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5687 } 5688 5689 static int 5690 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5691 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5692 bool *postponed_for_bulk) 5693 { 5694 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5695 5696 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5697 op_ctx_basic->ralue_pl); 5698 } 5699 5700 static bool 5701 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5702 { 5703 return true; 5704 } 5705 5706 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5707 struct mlxsw_sp_fib_entry *fib_entry, 5708 enum mlxsw_sp_fib_entry_op op) 5709 { 5710 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5711 5712 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5713 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5714 fib_entry->fib_node->key.prefix_len, 5715 fib_entry->fib_node->key.addr, 5716 fib_entry->priv); 5717 } 5718 5719 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5720 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5721 const struct mlxsw_sp_router_ll_ops *ll_ops) 5722 { 5723 bool postponed_for_bulk = false; 5724 int err; 5725 5726 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5727 if (!postponed_for_bulk) 5728 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5729 return err; 5730 } 5731 5732 static int mlxsw_sp_adj_discard_write(struct mlxsw_sp *mlxsw_sp) 5733 { 5734 enum mlxsw_reg_ratr_trap_action trap_action; 5735 char ratr_pl[MLXSW_REG_RATR_LEN]; 5736 int err; 5737 5738 if (mlxsw_sp->router->adj_discard_index_valid) 5739 return 0; 5740 5741 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5742 &mlxsw_sp->router->adj_discard_index); 5743 if (err) 5744 return err; 5745 5746 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 5747 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 5748 MLXSW_REG_RATR_TYPE_ETHERNET, 5749 mlxsw_sp->router->adj_discard_index, 5750 mlxsw_sp->router->lb_rif_index); 5751 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 5752 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 5753 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5754 if (err) 5755 goto err_ratr_write; 5756 5757 mlxsw_sp->router->adj_discard_index_valid = true; 5758 5759 return 0; 5760 5761 err_ratr_write: 5762 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 5763 mlxsw_sp->router->adj_discard_index); 5764 return err; 5765 } 5766 5767 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5768 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5769 struct mlxsw_sp_fib_entry *fib_entry, 5770 enum mlxsw_sp_fib_entry_op op) 5771 { 5772 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5773 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5774 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5775 enum mlxsw_reg_ralue_trap_action trap_action; 5776 u16 trap_id = 0; 5777 u32 adjacency_index = 0; 5778 u16 ecmp_size = 0; 5779 int err; 5780 5781 /* In case the nexthop group adjacency index is valid, use it 5782 * with provided ECMP size. Otherwise, setup trap and pass 5783 * traffic to kernel. 5784 */ 5785 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5786 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5787 adjacency_index = nhgi->adj_index; 5788 ecmp_size = nhgi->ecmp_size; 5789 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5790 err = mlxsw_sp_adj_discard_write(mlxsw_sp); 5791 if (err) 5792 return err; 5793 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5794 adjacency_index = mlxsw_sp->router->adj_discard_index; 5795 ecmp_size = 1; 5796 } else { 5797 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5798 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5799 } 5800 5801 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5802 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5803 adjacency_index, ecmp_size); 5804 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5805 } 5806 5807 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5808 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5809 struct mlxsw_sp_fib_entry *fib_entry, 5810 enum mlxsw_sp_fib_entry_op op) 5811 { 5812 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5813 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5814 enum mlxsw_reg_ralue_trap_action trap_action; 5815 u16 trap_id = 0; 5816 u16 rif_index = 0; 5817 5818 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5819 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5820 rif_index = rif->rif_index; 5821 } else { 5822 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5823 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5824 } 5825 5826 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5827 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5828 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5829 } 5830 5831 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5832 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5833 struct mlxsw_sp_fib_entry *fib_entry, 5834 enum mlxsw_sp_fib_entry_op op) 5835 { 5836 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5837 5838 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5839 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5840 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5841 } 5842 5843 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5844 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5845 struct mlxsw_sp_fib_entry *fib_entry, 5846 enum mlxsw_sp_fib_entry_op op) 5847 { 5848 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5849 enum mlxsw_reg_ralue_trap_action trap_action; 5850 5851 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5852 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5853 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 5854 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5855 } 5856 5857 static int 5858 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5859 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5860 struct mlxsw_sp_fib_entry *fib_entry, 5861 enum mlxsw_sp_fib_entry_op op) 5862 { 5863 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5864 enum mlxsw_reg_ralue_trap_action trap_action; 5865 u16 trap_id; 5866 5867 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5868 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5869 5870 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5871 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 5872 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5873 } 5874 5875 static int 5876 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5877 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5878 struct mlxsw_sp_fib_entry *fib_entry, 5879 enum mlxsw_sp_fib_entry_op op) 5880 { 5881 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5882 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5883 const struct mlxsw_sp_ipip_ops *ipip_ops; 5884 int err; 5885 5886 if (WARN_ON(!ipip_entry)) 5887 return -EINVAL; 5888 5889 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5890 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5891 fib_entry->decap.tunnel_index); 5892 if (err) 5893 return err; 5894 5895 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5896 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5897 fib_entry->decap.tunnel_index); 5898 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5899 } 5900 5901 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5902 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5903 struct mlxsw_sp_fib_entry *fib_entry, 5904 enum mlxsw_sp_fib_entry_op op) 5905 { 5906 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5907 5908 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5909 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 5910 fib_entry->decap.tunnel_index); 5911 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5912 } 5913 5914 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5915 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5916 struct mlxsw_sp_fib_entry *fib_entry, 5917 enum mlxsw_sp_fib_entry_op op) 5918 { 5919 switch (fib_entry->type) { 5920 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5921 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 5922 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5923 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 5924 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5925 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 5926 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5927 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 5928 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5929 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 5930 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5931 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 5932 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5933 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 5934 } 5935 return -EINVAL; 5936 } 5937 5938 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5939 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5940 struct mlxsw_sp_fib_entry *fib_entry, 5941 enum mlxsw_sp_fib_entry_op op) 5942 { 5943 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 5944 5945 if (err) 5946 return err; 5947 5948 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5949 5950 return err; 5951 } 5952 5953 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5954 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5955 struct mlxsw_sp_fib_entry *fib_entry, 5956 bool is_new) 5957 { 5958 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5959 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 5960 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 5961 } 5962 5963 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5964 struct mlxsw_sp_fib_entry *fib_entry) 5965 { 5966 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 5967 5968 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 5969 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 5970 } 5971 5972 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5973 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5974 struct mlxsw_sp_fib_entry *fib_entry) 5975 { 5976 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5977 5978 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 5979 return 0; 5980 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 5981 MLXSW_SP_FIB_ENTRY_OP_DELETE); 5982 } 5983 5984 static int 5985 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5986 const struct fib_entry_notifier_info *fen_info, 5987 struct mlxsw_sp_fib_entry *fib_entry) 5988 { 5989 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5990 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5991 struct mlxsw_sp_router *router = mlxsw_sp->router; 5992 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5993 int ifindex = nhgi->nexthops[0].ifindex; 5994 struct mlxsw_sp_ipip_entry *ipip_entry; 5995 5996 switch (fen_info->type) { 5997 case RTN_LOCAL: 5998 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5999 MLXSW_SP_L3_PROTO_IPV4, dip); 6000 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6001 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6002 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6003 fib_entry, 6004 ipip_entry); 6005 } 6006 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6007 MLXSW_SP_L3_PROTO_IPV4, 6008 &dip)) { 6009 u32 tunnel_index; 6010 6011 tunnel_index = router->nve_decap_config.tunnel_index; 6012 fib_entry->decap.tunnel_index = tunnel_index; 6013 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6014 return 0; 6015 } 6016 fallthrough; 6017 case RTN_BROADCAST: 6018 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6019 return 0; 6020 case RTN_BLACKHOLE: 6021 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6022 return 0; 6023 case RTN_UNREACHABLE: 6024 case RTN_PROHIBIT: 6025 /* Packets hitting these routes need to be trapped, but 6026 * can do so with a lower priority than packets directed 6027 * at the host, so use action type local instead of trap. 6028 */ 6029 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6030 return 0; 6031 case RTN_UNICAST: 6032 if (nhgi->gateway) 6033 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6034 else 6035 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6036 return 0; 6037 default: 6038 return -EINVAL; 6039 } 6040 } 6041 6042 static void 6043 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6044 struct mlxsw_sp_fib_entry *fib_entry) 6045 { 6046 switch (fib_entry->type) { 6047 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6048 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6049 break; 6050 default: 6051 break; 6052 } 6053 } 6054 6055 static struct mlxsw_sp_fib4_entry * 6056 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6057 struct mlxsw_sp_fib_node *fib_node, 6058 const struct fib_entry_notifier_info *fen_info) 6059 { 6060 struct mlxsw_sp_fib4_entry *fib4_entry; 6061 struct mlxsw_sp_fib_entry *fib_entry; 6062 int err; 6063 6064 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6065 if (!fib4_entry) 6066 return ERR_PTR(-ENOMEM); 6067 fib_entry = &fib4_entry->common; 6068 6069 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6070 if (IS_ERR(fib_entry->priv)) { 6071 err = PTR_ERR(fib_entry->priv); 6072 goto err_fib_entry_priv_create; 6073 } 6074 6075 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6076 if (err) 6077 goto err_nexthop4_group_get; 6078 6079 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6080 fib_node->fib); 6081 if (err) 6082 goto err_nexthop_group_vr_link; 6083 6084 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6085 if (err) 6086 goto err_fib4_entry_type_set; 6087 6088 fib4_entry->fi = fen_info->fi; 6089 fib_info_hold(fib4_entry->fi); 6090 fib4_entry->tb_id = fen_info->tb_id; 6091 fib4_entry->type = fen_info->type; 6092 fib4_entry->tos = fen_info->tos; 6093 6094 fib_entry->fib_node = fib_node; 6095 6096 return fib4_entry; 6097 6098 err_fib4_entry_type_set: 6099 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6100 err_nexthop_group_vr_link: 6101 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6102 err_nexthop4_group_get: 6103 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6104 err_fib_entry_priv_create: 6105 kfree(fib4_entry); 6106 return ERR_PTR(err); 6107 } 6108 6109 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6110 struct mlxsw_sp_fib4_entry *fib4_entry) 6111 { 6112 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6113 6114 fib_info_put(fib4_entry->fi); 6115 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6116 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6117 fib_node->fib); 6118 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6119 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 6120 kfree(fib4_entry); 6121 } 6122 6123 static struct mlxsw_sp_fib4_entry * 6124 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6125 const struct fib_entry_notifier_info *fen_info) 6126 { 6127 struct mlxsw_sp_fib4_entry *fib4_entry; 6128 struct mlxsw_sp_fib_node *fib_node; 6129 struct mlxsw_sp_fib *fib; 6130 struct mlxsw_sp_vr *vr; 6131 6132 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6133 if (!vr) 6134 return NULL; 6135 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6136 6137 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6138 sizeof(fen_info->dst), 6139 fen_info->dst_len); 6140 if (!fib_node) 6141 return NULL; 6142 6143 fib4_entry = container_of(fib_node->fib_entry, 6144 struct mlxsw_sp_fib4_entry, common); 6145 if (fib4_entry->tb_id == fen_info->tb_id && 6146 fib4_entry->tos == fen_info->tos && 6147 fib4_entry->type == fen_info->type && 6148 fib4_entry->fi == fen_info->fi) 6149 return fib4_entry; 6150 6151 return NULL; 6152 } 6153 6154 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6155 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6156 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6157 .key_len = sizeof(struct mlxsw_sp_fib_key), 6158 .automatic_shrinking = true, 6159 }; 6160 6161 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6162 struct mlxsw_sp_fib_node *fib_node) 6163 { 6164 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6165 mlxsw_sp_fib_ht_params); 6166 } 6167 6168 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6169 struct mlxsw_sp_fib_node *fib_node) 6170 { 6171 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6172 mlxsw_sp_fib_ht_params); 6173 } 6174 6175 static struct mlxsw_sp_fib_node * 6176 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6177 size_t addr_len, unsigned char prefix_len) 6178 { 6179 struct mlxsw_sp_fib_key key; 6180 6181 memset(&key, 0, sizeof(key)); 6182 memcpy(key.addr, addr, addr_len); 6183 key.prefix_len = prefix_len; 6184 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6185 } 6186 6187 static struct mlxsw_sp_fib_node * 6188 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6189 size_t addr_len, unsigned char prefix_len) 6190 { 6191 struct mlxsw_sp_fib_node *fib_node; 6192 6193 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6194 if (!fib_node) 6195 return NULL; 6196 6197 list_add(&fib_node->list, &fib->node_list); 6198 memcpy(fib_node->key.addr, addr, addr_len); 6199 fib_node->key.prefix_len = prefix_len; 6200 6201 return fib_node; 6202 } 6203 6204 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6205 { 6206 list_del(&fib_node->list); 6207 kfree(fib_node); 6208 } 6209 6210 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6211 struct mlxsw_sp_fib_node *fib_node) 6212 { 6213 struct mlxsw_sp_prefix_usage req_prefix_usage; 6214 struct mlxsw_sp_fib *fib = fib_node->fib; 6215 struct mlxsw_sp_lpm_tree *lpm_tree; 6216 int err; 6217 6218 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6219 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6220 goto out; 6221 6222 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6223 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6224 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6225 fib->proto); 6226 if (IS_ERR(lpm_tree)) 6227 return PTR_ERR(lpm_tree); 6228 6229 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6230 if (err) 6231 goto err_lpm_tree_replace; 6232 6233 out: 6234 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6235 return 0; 6236 6237 err_lpm_tree_replace: 6238 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6239 return err; 6240 } 6241 6242 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6243 struct mlxsw_sp_fib_node *fib_node) 6244 { 6245 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6246 struct mlxsw_sp_prefix_usage req_prefix_usage; 6247 struct mlxsw_sp_fib *fib = fib_node->fib; 6248 int err; 6249 6250 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6251 return; 6252 /* Try to construct a new LPM tree from the current prefix usage 6253 * minus the unused one. If we fail, continue using the old one. 6254 */ 6255 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6256 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6257 fib_node->key.prefix_len); 6258 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6259 fib->proto); 6260 if (IS_ERR(lpm_tree)) 6261 return; 6262 6263 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6264 if (err) 6265 goto err_lpm_tree_replace; 6266 6267 return; 6268 6269 err_lpm_tree_replace: 6270 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6271 } 6272 6273 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6274 struct mlxsw_sp_fib_node *fib_node, 6275 struct mlxsw_sp_fib *fib) 6276 { 6277 int err; 6278 6279 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6280 if (err) 6281 return err; 6282 fib_node->fib = fib; 6283 6284 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6285 if (err) 6286 goto err_fib_lpm_tree_link; 6287 6288 return 0; 6289 6290 err_fib_lpm_tree_link: 6291 fib_node->fib = NULL; 6292 mlxsw_sp_fib_node_remove(fib, fib_node); 6293 return err; 6294 } 6295 6296 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6297 struct mlxsw_sp_fib_node *fib_node) 6298 { 6299 struct mlxsw_sp_fib *fib = fib_node->fib; 6300 6301 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6302 fib_node->fib = NULL; 6303 mlxsw_sp_fib_node_remove(fib, fib_node); 6304 } 6305 6306 static struct mlxsw_sp_fib_node * 6307 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6308 size_t addr_len, unsigned char prefix_len, 6309 enum mlxsw_sp_l3proto proto) 6310 { 6311 struct mlxsw_sp_fib_node *fib_node; 6312 struct mlxsw_sp_fib *fib; 6313 struct mlxsw_sp_vr *vr; 6314 int err; 6315 6316 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6317 if (IS_ERR(vr)) 6318 return ERR_CAST(vr); 6319 fib = mlxsw_sp_vr_fib(vr, proto); 6320 6321 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6322 if (fib_node) 6323 return fib_node; 6324 6325 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6326 if (!fib_node) { 6327 err = -ENOMEM; 6328 goto err_fib_node_create; 6329 } 6330 6331 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6332 if (err) 6333 goto err_fib_node_init; 6334 6335 return fib_node; 6336 6337 err_fib_node_init: 6338 mlxsw_sp_fib_node_destroy(fib_node); 6339 err_fib_node_create: 6340 mlxsw_sp_vr_put(mlxsw_sp, vr); 6341 return ERR_PTR(err); 6342 } 6343 6344 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6345 struct mlxsw_sp_fib_node *fib_node) 6346 { 6347 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6348 6349 if (fib_node->fib_entry) 6350 return; 6351 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6352 mlxsw_sp_fib_node_destroy(fib_node); 6353 mlxsw_sp_vr_put(mlxsw_sp, vr); 6354 } 6355 6356 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6357 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6358 struct mlxsw_sp_fib_entry *fib_entry) 6359 { 6360 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6361 bool is_new = !fib_node->fib_entry; 6362 int err; 6363 6364 fib_node->fib_entry = fib_entry; 6365 6366 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 6367 if (err) 6368 goto err_fib_entry_update; 6369 6370 return 0; 6371 6372 err_fib_entry_update: 6373 fib_node->fib_entry = NULL; 6374 return err; 6375 } 6376 6377 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6378 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6379 struct mlxsw_sp_fib_entry *fib_entry) 6380 { 6381 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6382 int err; 6383 6384 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 6385 fib_node->fib_entry = NULL; 6386 return err; 6387 } 6388 6389 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6390 struct mlxsw_sp_fib_entry *fib_entry) 6391 { 6392 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6393 6394 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6395 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 6396 } 6397 6398 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6399 { 6400 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6401 struct mlxsw_sp_fib4_entry *fib4_replaced; 6402 6403 if (!fib_node->fib_entry) 6404 return true; 6405 6406 fib4_replaced = container_of(fib_node->fib_entry, 6407 struct mlxsw_sp_fib4_entry, common); 6408 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6409 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6410 return false; 6411 6412 return true; 6413 } 6414 6415 static int 6416 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6417 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6418 const struct fib_entry_notifier_info *fen_info) 6419 { 6420 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6421 struct mlxsw_sp_fib_entry *replaced; 6422 struct mlxsw_sp_fib_node *fib_node; 6423 int err; 6424 6425 if (mlxsw_sp->router->aborted) 6426 return 0; 6427 6428 if (fen_info->fi->nh && 6429 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6430 return 0; 6431 6432 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6433 &fen_info->dst, sizeof(fen_info->dst), 6434 fen_info->dst_len, 6435 MLXSW_SP_L3_PROTO_IPV4); 6436 if (IS_ERR(fib_node)) { 6437 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6438 return PTR_ERR(fib_node); 6439 } 6440 6441 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6442 if (IS_ERR(fib4_entry)) { 6443 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6444 err = PTR_ERR(fib4_entry); 6445 goto err_fib4_entry_create; 6446 } 6447 6448 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6449 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6450 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6451 return 0; 6452 } 6453 6454 replaced = fib_node->fib_entry; 6455 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 6456 if (err) { 6457 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6458 goto err_fib_node_entry_link; 6459 } 6460 6461 /* Nothing to replace */ 6462 if (!replaced) 6463 return 0; 6464 6465 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6466 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6467 common); 6468 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6469 6470 return 0; 6471 6472 err_fib_node_entry_link: 6473 fib_node->fib_entry = replaced; 6474 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6475 err_fib4_entry_create: 6476 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6477 return err; 6478 } 6479 6480 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6481 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6482 struct fib_entry_notifier_info *fen_info) 6483 { 6484 struct mlxsw_sp_fib4_entry *fib4_entry; 6485 struct mlxsw_sp_fib_node *fib_node; 6486 int err; 6487 6488 if (mlxsw_sp->router->aborted) 6489 return 0; 6490 6491 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6492 if (!fib4_entry) 6493 return 0; 6494 fib_node = fib4_entry->common.fib_node; 6495 6496 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6497 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6498 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6499 return err; 6500 } 6501 6502 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6503 { 6504 /* Multicast routes aren't supported, so ignore them. Neighbour 6505 * Discovery packets are specifically trapped. 6506 */ 6507 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6508 return true; 6509 6510 /* Cloned routes are irrelevant in the forwarding path. */ 6511 if (rt->fib6_flags & RTF_CACHE) 6512 return true; 6513 6514 return false; 6515 } 6516 6517 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6518 { 6519 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6520 6521 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6522 if (!mlxsw_sp_rt6) 6523 return ERR_PTR(-ENOMEM); 6524 6525 /* In case of route replace, replaced route is deleted with 6526 * no notification. Take reference to prevent accessing freed 6527 * memory. 6528 */ 6529 mlxsw_sp_rt6->rt = rt; 6530 fib6_info_hold(rt); 6531 6532 return mlxsw_sp_rt6; 6533 } 6534 6535 #if IS_ENABLED(CONFIG_IPV6) 6536 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6537 { 6538 fib6_info_release(rt); 6539 } 6540 #else 6541 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6542 { 6543 } 6544 #endif 6545 6546 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6547 { 6548 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6549 6550 if (!mlxsw_sp_rt6->rt->nh) 6551 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6552 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6553 kfree(mlxsw_sp_rt6); 6554 } 6555 6556 static struct fib6_info * 6557 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6558 { 6559 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6560 list)->rt; 6561 } 6562 6563 static struct mlxsw_sp_rt6 * 6564 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6565 const struct fib6_info *rt) 6566 { 6567 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6568 6569 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6570 if (mlxsw_sp_rt6->rt == rt) 6571 return mlxsw_sp_rt6; 6572 } 6573 6574 return NULL; 6575 } 6576 6577 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6578 const struct fib6_info *rt, 6579 enum mlxsw_sp_ipip_type *ret) 6580 { 6581 return rt->fib6_nh->fib_nh_dev && 6582 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6583 } 6584 6585 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6586 struct mlxsw_sp_nexthop_group *nh_grp, 6587 struct mlxsw_sp_nexthop *nh, 6588 const struct fib6_info *rt) 6589 { 6590 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6591 6592 nh->nhgi = nh_grp->nhgi; 6593 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6594 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6595 #if IS_ENABLED(CONFIG_IPV6) 6596 nh->neigh_tbl = &nd_tbl; 6597 #endif 6598 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6599 6600 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6601 6602 if (!dev) 6603 return 0; 6604 nh->ifindex = dev->ifindex; 6605 6606 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6607 } 6608 6609 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6610 struct mlxsw_sp_nexthop *nh) 6611 { 6612 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6613 list_del(&nh->router_list_node); 6614 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6615 } 6616 6617 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6618 const struct fib6_info *rt) 6619 { 6620 return rt->fib6_nh->fib_nh_gw_family || 6621 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6622 } 6623 6624 static int 6625 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6626 struct mlxsw_sp_nexthop_group *nh_grp, 6627 struct mlxsw_sp_fib6_entry *fib6_entry) 6628 { 6629 struct mlxsw_sp_nexthop_group_info *nhgi; 6630 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6631 struct mlxsw_sp_nexthop *nh; 6632 int err, i; 6633 6634 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6635 GFP_KERNEL); 6636 if (!nhgi) 6637 return -ENOMEM; 6638 nh_grp->nhgi = nhgi; 6639 nhgi->nh_grp = nh_grp; 6640 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6641 struct mlxsw_sp_rt6, list); 6642 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6643 nhgi->count = fib6_entry->nrt6; 6644 for (i = 0; i < nhgi->count; i++) { 6645 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6646 6647 nh = &nhgi->nexthops[i]; 6648 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6649 if (err) 6650 goto err_nexthop6_init; 6651 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6652 } 6653 nh_grp->nhgi = nhgi; 6654 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6655 if (err) 6656 goto err_group_refresh; 6657 6658 return 0; 6659 6660 err_group_refresh: 6661 i = nhgi->count; 6662 err_nexthop6_init: 6663 for (i--; i >= 0; i--) { 6664 nh = &nhgi->nexthops[i]; 6665 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6666 } 6667 kfree(nhgi); 6668 return err; 6669 } 6670 6671 static void 6672 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6673 struct mlxsw_sp_nexthop_group *nh_grp) 6674 { 6675 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6676 int i; 6677 6678 for (i = nhgi->count - 1; i >= 0; i--) { 6679 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6680 6681 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6682 } 6683 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6684 WARN_ON_ONCE(nhgi->adj_index_valid); 6685 kfree(nhgi); 6686 } 6687 6688 static struct mlxsw_sp_nexthop_group * 6689 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6690 struct mlxsw_sp_fib6_entry *fib6_entry) 6691 { 6692 struct mlxsw_sp_nexthop_group *nh_grp; 6693 int err; 6694 6695 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6696 if (!nh_grp) 6697 return ERR_PTR(-ENOMEM); 6698 INIT_LIST_HEAD(&nh_grp->vr_list); 6699 err = rhashtable_init(&nh_grp->vr_ht, 6700 &mlxsw_sp_nexthop_group_vr_ht_params); 6701 if (err) 6702 goto err_nexthop_group_vr_ht_init; 6703 INIT_LIST_HEAD(&nh_grp->fib_list); 6704 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6705 6706 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6707 if (err) 6708 goto err_nexthop_group_info_init; 6709 6710 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6711 if (err) 6712 goto err_nexthop_group_insert; 6713 6714 nh_grp->can_destroy = true; 6715 6716 return nh_grp; 6717 6718 err_nexthop_group_insert: 6719 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6720 err_nexthop_group_info_init: 6721 rhashtable_destroy(&nh_grp->vr_ht); 6722 err_nexthop_group_vr_ht_init: 6723 kfree(nh_grp); 6724 return ERR_PTR(err); 6725 } 6726 6727 static void 6728 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6729 struct mlxsw_sp_nexthop_group *nh_grp) 6730 { 6731 if (!nh_grp->can_destroy) 6732 return; 6733 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6734 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6735 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6736 rhashtable_destroy(&nh_grp->vr_ht); 6737 kfree(nh_grp); 6738 } 6739 6740 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6741 struct mlxsw_sp_fib6_entry *fib6_entry) 6742 { 6743 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6744 struct mlxsw_sp_nexthop_group *nh_grp; 6745 6746 if (rt->nh) { 6747 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6748 rt->nh->id); 6749 if (WARN_ON_ONCE(!nh_grp)) 6750 return -EINVAL; 6751 goto out; 6752 } 6753 6754 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6755 if (!nh_grp) { 6756 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6757 if (IS_ERR(nh_grp)) 6758 return PTR_ERR(nh_grp); 6759 } 6760 6761 /* The route and the nexthop are described by the same struct, so we 6762 * need to the update the nexthop offload indication for the new route. 6763 */ 6764 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6765 6766 out: 6767 list_add_tail(&fib6_entry->common.nexthop_group_node, 6768 &nh_grp->fib_list); 6769 fib6_entry->common.nh_group = nh_grp; 6770 6771 return 0; 6772 } 6773 6774 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6775 struct mlxsw_sp_fib_entry *fib_entry) 6776 { 6777 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6778 6779 list_del(&fib_entry->nexthop_group_node); 6780 if (!list_empty(&nh_grp->fib_list)) 6781 return; 6782 6783 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6784 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6785 return; 6786 } 6787 6788 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6789 } 6790 6791 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6792 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6793 struct mlxsw_sp_fib6_entry *fib6_entry) 6794 { 6795 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6796 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6797 int err; 6798 6799 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6800 fib6_entry->common.nh_group = NULL; 6801 list_del(&fib6_entry->common.nexthop_group_node); 6802 6803 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6804 if (err) 6805 goto err_nexthop6_group_get; 6806 6807 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6808 fib_node->fib); 6809 if (err) 6810 goto err_nexthop_group_vr_link; 6811 6812 /* In case this entry is offloaded, then the adjacency index 6813 * currently associated with it in the device's table is that 6814 * of the old group. Start using the new one instead. 6815 */ 6816 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6817 &fib6_entry->common, false); 6818 if (err) 6819 goto err_fib_entry_update; 6820 6821 if (list_empty(&old_nh_grp->fib_list)) 6822 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6823 6824 return 0; 6825 6826 err_fib_entry_update: 6827 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6828 fib_node->fib); 6829 err_nexthop_group_vr_link: 6830 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6831 err_nexthop6_group_get: 6832 list_add_tail(&fib6_entry->common.nexthop_group_node, 6833 &old_nh_grp->fib_list); 6834 fib6_entry->common.nh_group = old_nh_grp; 6835 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6836 return err; 6837 } 6838 6839 static int 6840 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6841 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6842 struct mlxsw_sp_fib6_entry *fib6_entry, 6843 struct fib6_info **rt_arr, unsigned int nrt6) 6844 { 6845 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6846 int err, i; 6847 6848 for (i = 0; i < nrt6; i++) { 6849 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6850 if (IS_ERR(mlxsw_sp_rt6)) { 6851 err = PTR_ERR(mlxsw_sp_rt6); 6852 goto err_rt6_create; 6853 } 6854 6855 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6856 fib6_entry->nrt6++; 6857 } 6858 6859 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6860 if (err) 6861 goto err_nexthop6_group_update; 6862 6863 return 0; 6864 6865 err_nexthop6_group_update: 6866 i = nrt6; 6867 err_rt6_create: 6868 for (i--; i >= 0; i--) { 6869 fib6_entry->nrt6--; 6870 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6871 struct mlxsw_sp_rt6, list); 6872 list_del(&mlxsw_sp_rt6->list); 6873 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6874 } 6875 return err; 6876 } 6877 6878 static void 6879 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6880 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6881 struct mlxsw_sp_fib6_entry *fib6_entry, 6882 struct fib6_info **rt_arr, unsigned int nrt6) 6883 { 6884 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6885 int i; 6886 6887 for (i = 0; i < nrt6; i++) { 6888 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6889 rt_arr[i]); 6890 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6891 continue; 6892 6893 fib6_entry->nrt6--; 6894 list_del(&mlxsw_sp_rt6->list); 6895 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6896 } 6897 6898 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 6899 } 6900 6901 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6902 struct mlxsw_sp_fib_entry *fib_entry, 6903 const struct fib6_info *rt) 6904 { 6905 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST)) 6906 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6907 else if (rt->fib6_type == RTN_BLACKHOLE) 6908 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6909 else if (rt->fib6_flags & RTF_REJECT) 6910 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6911 else if (fib_entry->nh_group->nhgi->gateway) 6912 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6913 else 6914 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6915 } 6916 6917 static void 6918 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6919 { 6920 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6921 6922 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6923 list) { 6924 fib6_entry->nrt6--; 6925 list_del(&mlxsw_sp_rt6->list); 6926 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6927 } 6928 } 6929 6930 static struct mlxsw_sp_fib6_entry * 6931 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6932 struct mlxsw_sp_fib_node *fib_node, 6933 struct fib6_info **rt_arr, unsigned int nrt6) 6934 { 6935 struct mlxsw_sp_fib6_entry *fib6_entry; 6936 struct mlxsw_sp_fib_entry *fib_entry; 6937 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6938 int err, i; 6939 6940 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6941 if (!fib6_entry) 6942 return ERR_PTR(-ENOMEM); 6943 fib_entry = &fib6_entry->common; 6944 6945 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6946 if (IS_ERR(fib_entry->priv)) { 6947 err = PTR_ERR(fib_entry->priv); 6948 goto err_fib_entry_priv_create; 6949 } 6950 6951 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6952 6953 for (i = 0; i < nrt6; i++) { 6954 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6955 if (IS_ERR(mlxsw_sp_rt6)) { 6956 err = PTR_ERR(mlxsw_sp_rt6); 6957 goto err_rt6_create; 6958 } 6959 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6960 fib6_entry->nrt6++; 6961 } 6962 6963 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6964 if (err) 6965 goto err_nexthop6_group_get; 6966 6967 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6968 fib_node->fib); 6969 if (err) 6970 goto err_nexthop_group_vr_link; 6971 6972 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6973 6974 fib_entry->fib_node = fib_node; 6975 6976 return fib6_entry; 6977 6978 err_nexthop_group_vr_link: 6979 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6980 err_nexthop6_group_get: 6981 i = nrt6; 6982 err_rt6_create: 6983 for (i--; i >= 0; i--) { 6984 fib6_entry->nrt6--; 6985 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6986 struct mlxsw_sp_rt6, list); 6987 list_del(&mlxsw_sp_rt6->list); 6988 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6989 } 6990 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6991 err_fib_entry_priv_create: 6992 kfree(fib6_entry); 6993 return ERR_PTR(err); 6994 } 6995 6996 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6997 struct mlxsw_sp_fib6_entry *fib6_entry) 6998 { 6999 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7000 7001 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7002 fib_node->fib); 7003 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7004 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7005 WARN_ON(fib6_entry->nrt6); 7006 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 7007 kfree(fib6_entry); 7008 } 7009 7010 static struct mlxsw_sp_fib6_entry * 7011 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7012 const struct fib6_info *rt) 7013 { 7014 struct mlxsw_sp_fib6_entry *fib6_entry; 7015 struct mlxsw_sp_fib_node *fib_node; 7016 struct mlxsw_sp_fib *fib; 7017 struct fib6_info *cmp_rt; 7018 struct mlxsw_sp_vr *vr; 7019 7020 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7021 if (!vr) 7022 return NULL; 7023 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7024 7025 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7026 sizeof(rt->fib6_dst.addr), 7027 rt->fib6_dst.plen); 7028 if (!fib_node) 7029 return NULL; 7030 7031 fib6_entry = container_of(fib_node->fib_entry, 7032 struct mlxsw_sp_fib6_entry, common); 7033 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7034 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7035 rt->fib6_metric == cmp_rt->fib6_metric && 7036 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7037 return fib6_entry; 7038 7039 return NULL; 7040 } 7041 7042 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7043 { 7044 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7045 struct mlxsw_sp_fib6_entry *fib6_replaced; 7046 struct fib6_info *rt, *rt_replaced; 7047 7048 if (!fib_node->fib_entry) 7049 return true; 7050 7051 fib6_replaced = container_of(fib_node->fib_entry, 7052 struct mlxsw_sp_fib6_entry, 7053 common); 7054 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7055 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7056 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7057 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7058 return false; 7059 7060 return true; 7061 } 7062 7063 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7064 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7065 struct fib6_info **rt_arr, unsigned int nrt6) 7066 { 7067 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7068 struct mlxsw_sp_fib_entry *replaced; 7069 struct mlxsw_sp_fib_node *fib_node; 7070 struct fib6_info *rt = rt_arr[0]; 7071 int err; 7072 7073 if (mlxsw_sp->router->aborted) 7074 return 0; 7075 7076 if (rt->fib6_src.plen) 7077 return -EINVAL; 7078 7079 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7080 return 0; 7081 7082 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7083 return 0; 7084 7085 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7086 &rt->fib6_dst.addr, 7087 sizeof(rt->fib6_dst.addr), 7088 rt->fib6_dst.plen, 7089 MLXSW_SP_L3_PROTO_IPV6); 7090 if (IS_ERR(fib_node)) 7091 return PTR_ERR(fib_node); 7092 7093 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7094 nrt6); 7095 if (IS_ERR(fib6_entry)) { 7096 err = PTR_ERR(fib6_entry); 7097 goto err_fib6_entry_create; 7098 } 7099 7100 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7101 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7102 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7103 return 0; 7104 } 7105 7106 replaced = fib_node->fib_entry; 7107 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 7108 if (err) 7109 goto err_fib_node_entry_link; 7110 7111 /* Nothing to replace */ 7112 if (!replaced) 7113 return 0; 7114 7115 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7116 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7117 common); 7118 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7119 7120 return 0; 7121 7122 err_fib_node_entry_link: 7123 fib_node->fib_entry = replaced; 7124 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7125 err_fib6_entry_create: 7126 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7127 return err; 7128 } 7129 7130 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7131 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7132 struct fib6_info **rt_arr, unsigned int nrt6) 7133 { 7134 struct mlxsw_sp_fib6_entry *fib6_entry; 7135 struct mlxsw_sp_fib_node *fib_node; 7136 struct fib6_info *rt = rt_arr[0]; 7137 int err; 7138 7139 if (mlxsw_sp->router->aborted) 7140 return 0; 7141 7142 if (rt->fib6_src.plen) 7143 return -EINVAL; 7144 7145 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7146 return 0; 7147 7148 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7149 &rt->fib6_dst.addr, 7150 sizeof(rt->fib6_dst.addr), 7151 rt->fib6_dst.plen, 7152 MLXSW_SP_L3_PROTO_IPV6); 7153 if (IS_ERR(fib_node)) 7154 return PTR_ERR(fib_node); 7155 7156 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7157 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7158 return -EINVAL; 7159 } 7160 7161 fib6_entry = container_of(fib_node->fib_entry, 7162 struct mlxsw_sp_fib6_entry, common); 7163 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7164 if (err) 7165 goto err_fib6_entry_nexthop_add; 7166 7167 return 0; 7168 7169 err_fib6_entry_nexthop_add: 7170 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7171 return err; 7172 } 7173 7174 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7175 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7176 struct fib6_info **rt_arr, unsigned int nrt6) 7177 { 7178 struct mlxsw_sp_fib6_entry *fib6_entry; 7179 struct mlxsw_sp_fib_node *fib_node; 7180 struct fib6_info *rt = rt_arr[0]; 7181 int err; 7182 7183 if (mlxsw_sp->router->aborted) 7184 return 0; 7185 7186 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7187 return 0; 7188 7189 /* Multipath routes are first added to the FIB trie and only then 7190 * notified. If we vetoed the addition, we will get a delete 7191 * notification for a route we do not have. Therefore, do not warn if 7192 * route was not found. 7193 */ 7194 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7195 if (!fib6_entry) 7196 return 0; 7197 7198 /* If not all the nexthops are deleted, then only reduce the nexthop 7199 * group. 7200 */ 7201 if (nrt6 != fib6_entry->nrt6) { 7202 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7203 return 0; 7204 } 7205 7206 fib_node = fib6_entry->common.fib_node; 7207 7208 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 7209 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7210 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7211 return err; 7212 } 7213 7214 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp, 7215 enum mlxsw_sp_l3proto proto, 7216 u8 tree_id) 7217 { 7218 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 7219 enum mlxsw_reg_ralxx_protocol ralxx_proto = 7220 (enum mlxsw_reg_ralxx_protocol) proto; 7221 struct mlxsw_sp_fib_entry_priv *priv; 7222 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 7223 char xralst_pl[MLXSW_REG_XRALST_LEN]; 7224 int i, err; 7225 7226 mlxsw_reg_xralta_pack(xralta_pl, true, ralxx_proto, tree_id); 7227 err = ll_ops->ralta_write(mlxsw_sp, xralta_pl); 7228 if (err) 7229 return err; 7230 7231 mlxsw_reg_xralst_pack(xralst_pl, 0xff, tree_id); 7232 err = ll_ops->ralst_write(mlxsw_sp, xralst_pl); 7233 if (err) 7234 return err; 7235 7236 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7237 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 7238 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7239 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 7240 7241 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7242 mlxsw_reg_xraltb_pack(xraltb_pl, vr->id, ralxx_proto, tree_id); 7243 err = ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 7244 if (err) 7245 return err; 7246 7247 priv = mlxsw_sp_fib_entry_priv_create(ll_ops); 7248 if (IS_ERR(priv)) 7249 return PTR_ERR(priv); 7250 7251 ll_ops->fib_entry_pack(op_ctx, proto, MLXSW_SP_FIB_ENTRY_OP_WRITE, 7252 vr->id, 0, NULL, priv); 7253 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 7254 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, NULL); 7255 mlxsw_sp_fib_entry_priv_put(priv); 7256 if (err) 7257 return err; 7258 } 7259 7260 return 0; 7261 } 7262 7263 static struct mlxsw_sp_mr_table * 7264 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7265 { 7266 if (family == RTNL_FAMILY_IPMR) 7267 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7268 else 7269 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7270 } 7271 7272 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7273 struct mfc_entry_notifier_info *men_info, 7274 bool replace) 7275 { 7276 struct mlxsw_sp_mr_table *mrt; 7277 struct mlxsw_sp_vr *vr; 7278 7279 if (mlxsw_sp->router->aborted) 7280 return 0; 7281 7282 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7283 if (IS_ERR(vr)) 7284 return PTR_ERR(vr); 7285 7286 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7287 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7288 } 7289 7290 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7291 struct mfc_entry_notifier_info *men_info) 7292 { 7293 struct mlxsw_sp_mr_table *mrt; 7294 struct mlxsw_sp_vr *vr; 7295 7296 if (mlxsw_sp->router->aborted) 7297 return; 7298 7299 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7300 if (WARN_ON(!vr)) 7301 return; 7302 7303 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7304 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7305 mlxsw_sp_vr_put(mlxsw_sp, vr); 7306 } 7307 7308 static int 7309 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7310 struct vif_entry_notifier_info *ven_info) 7311 { 7312 struct mlxsw_sp_mr_table *mrt; 7313 struct mlxsw_sp_rif *rif; 7314 struct mlxsw_sp_vr *vr; 7315 7316 if (mlxsw_sp->router->aborted) 7317 return 0; 7318 7319 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7320 if (IS_ERR(vr)) 7321 return PTR_ERR(vr); 7322 7323 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7324 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7325 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7326 ven_info->vif_index, 7327 ven_info->vif_flags, rif); 7328 } 7329 7330 static void 7331 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7332 struct vif_entry_notifier_info *ven_info) 7333 { 7334 struct mlxsw_sp_mr_table *mrt; 7335 struct mlxsw_sp_vr *vr; 7336 7337 if (mlxsw_sp->router->aborted) 7338 return; 7339 7340 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7341 if (WARN_ON(!vr)) 7342 return; 7343 7344 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7345 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7346 mlxsw_sp_vr_put(mlxsw_sp, vr); 7347 } 7348 7349 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) 7350 { 7351 enum mlxsw_sp_l3proto proto = MLXSW_SP_L3_PROTO_IPV4; 7352 int err; 7353 7354 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 7355 MLXSW_SP_LPM_TREE_MIN); 7356 if (err) 7357 return err; 7358 7359 /* The multicast router code does not need an abort trap as by default, 7360 * packets that don't match any routes are trapped to the CPU. 7361 */ 7362 7363 proto = MLXSW_SP_L3_PROTO_IPV6; 7364 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto, 7365 MLXSW_SP_LPM_TREE_MIN + 1); 7366 } 7367 7368 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7369 struct mlxsw_sp_fib_node *fib_node) 7370 { 7371 struct mlxsw_sp_fib4_entry *fib4_entry; 7372 7373 fib4_entry = container_of(fib_node->fib_entry, 7374 struct mlxsw_sp_fib4_entry, common); 7375 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7376 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7377 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7378 } 7379 7380 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7381 struct mlxsw_sp_fib_node *fib_node) 7382 { 7383 struct mlxsw_sp_fib6_entry *fib6_entry; 7384 7385 fib6_entry = container_of(fib_node->fib_entry, 7386 struct mlxsw_sp_fib6_entry, common); 7387 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7388 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7389 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7390 } 7391 7392 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7393 struct mlxsw_sp_fib_node *fib_node) 7394 { 7395 switch (fib_node->fib->proto) { 7396 case MLXSW_SP_L3_PROTO_IPV4: 7397 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7398 break; 7399 case MLXSW_SP_L3_PROTO_IPV6: 7400 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7401 break; 7402 } 7403 } 7404 7405 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7406 struct mlxsw_sp_vr *vr, 7407 enum mlxsw_sp_l3proto proto) 7408 { 7409 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7410 struct mlxsw_sp_fib_node *fib_node, *tmp; 7411 7412 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7413 bool do_break = &tmp->list == &fib->node_list; 7414 7415 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7416 if (do_break) 7417 break; 7418 } 7419 } 7420 7421 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7422 { 7423 int i, j; 7424 7425 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7426 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7427 7428 if (!mlxsw_sp_vr_is_used(vr)) 7429 continue; 7430 7431 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7432 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7433 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7434 7435 /* If virtual router was only used for IPv4, then it's no 7436 * longer used. 7437 */ 7438 if (!mlxsw_sp_vr_is_used(vr)) 7439 continue; 7440 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7441 } 7442 7443 /* After flushing all the routes, it is not possible anyone is still 7444 * using the adjacency index that is discarding packets, so free it in 7445 * case it was allocated. 7446 */ 7447 if (!mlxsw_sp->router->adj_discard_index_valid) 7448 return; 7449 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 7450 mlxsw_sp->router->adj_discard_index); 7451 mlxsw_sp->router->adj_discard_index_valid = false; 7452 } 7453 7454 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp) 7455 { 7456 int err; 7457 7458 if (mlxsw_sp->router->aborted) 7459 return; 7460 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n"); 7461 mlxsw_sp_router_fib_flush(mlxsw_sp); 7462 mlxsw_sp->router->aborted = true; 7463 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp); 7464 if (err) 7465 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n"); 7466 } 7467 7468 struct mlxsw_sp_fib6_event { 7469 struct fib6_info **rt_arr; 7470 unsigned int nrt6; 7471 }; 7472 7473 struct mlxsw_sp_fib_event { 7474 struct list_head list; /* node in fib queue */ 7475 union { 7476 struct mlxsw_sp_fib6_event fib6_event; 7477 struct fib_entry_notifier_info fen_info; 7478 struct fib_rule_notifier_info fr_info; 7479 struct fib_nh_notifier_info fnh_info; 7480 struct mfc_entry_notifier_info men_info; 7481 struct vif_entry_notifier_info ven_info; 7482 }; 7483 struct mlxsw_sp *mlxsw_sp; 7484 unsigned long event; 7485 int family; 7486 }; 7487 7488 static int 7489 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7490 struct fib6_entry_notifier_info *fen6_info) 7491 { 7492 struct fib6_info *rt = fen6_info->rt; 7493 struct fib6_info **rt_arr; 7494 struct fib6_info *iter; 7495 unsigned int nrt6; 7496 int i = 0; 7497 7498 nrt6 = fen6_info->nsiblings + 1; 7499 7500 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7501 if (!rt_arr) 7502 return -ENOMEM; 7503 7504 fib6_event->rt_arr = rt_arr; 7505 fib6_event->nrt6 = nrt6; 7506 7507 rt_arr[0] = rt; 7508 fib6_info_hold(rt); 7509 7510 if (!fen6_info->nsiblings) 7511 return 0; 7512 7513 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7514 if (i == fen6_info->nsiblings) 7515 break; 7516 7517 rt_arr[i + 1] = iter; 7518 fib6_info_hold(iter); 7519 i++; 7520 } 7521 WARN_ON_ONCE(i != fen6_info->nsiblings); 7522 7523 return 0; 7524 } 7525 7526 static void 7527 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7528 { 7529 int i; 7530 7531 for (i = 0; i < fib6_event->nrt6; i++) 7532 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7533 kfree(fib6_event->rt_arr); 7534 } 7535 7536 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7537 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7538 struct mlxsw_sp_fib_event *fib_event) 7539 { 7540 int err; 7541 7542 mlxsw_sp_span_respin(mlxsw_sp); 7543 7544 switch (fib_event->event) { 7545 case FIB_EVENT_ENTRY_REPLACE: 7546 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7547 if (err) { 7548 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7549 mlxsw_sp_router_fib_abort(mlxsw_sp); 7550 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7551 &fib_event->fen_info); 7552 } 7553 fib_info_put(fib_event->fen_info.fi); 7554 break; 7555 case FIB_EVENT_ENTRY_DEL: 7556 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7557 if (err) 7558 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7559 fib_info_put(fib_event->fen_info.fi); 7560 break; 7561 case FIB_EVENT_NH_ADD: 7562 case FIB_EVENT_NH_DEL: 7563 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7564 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7565 break; 7566 } 7567 } 7568 7569 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7570 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7571 struct mlxsw_sp_fib_event *fib_event) 7572 { 7573 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7574 int err; 7575 7576 mlxsw_sp_span_respin(mlxsw_sp); 7577 7578 switch (fib_event->event) { 7579 case FIB_EVENT_ENTRY_REPLACE: 7580 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7581 fib_event->fib6_event.nrt6); 7582 if (err) { 7583 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7584 mlxsw_sp_router_fib_abort(mlxsw_sp); 7585 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7586 fib6_event->rt_arr, 7587 fib6_event->nrt6); 7588 } 7589 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7590 break; 7591 case FIB_EVENT_ENTRY_APPEND: 7592 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7593 fib_event->fib6_event.nrt6); 7594 if (err) { 7595 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7596 mlxsw_sp_router_fib_abort(mlxsw_sp); 7597 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7598 fib6_event->rt_arr, 7599 fib6_event->nrt6); 7600 } 7601 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7602 break; 7603 case FIB_EVENT_ENTRY_DEL: 7604 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7605 fib_event->fib6_event.nrt6); 7606 if (err) 7607 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7608 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7609 break; 7610 } 7611 } 7612 7613 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7614 struct mlxsw_sp_fib_event *fib_event) 7615 { 7616 bool replace; 7617 int err; 7618 7619 rtnl_lock(); 7620 mutex_lock(&mlxsw_sp->router->lock); 7621 switch (fib_event->event) { 7622 case FIB_EVENT_ENTRY_REPLACE: 7623 case FIB_EVENT_ENTRY_ADD: 7624 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7625 7626 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7627 if (err) 7628 mlxsw_sp_router_fib_abort(mlxsw_sp); 7629 mr_cache_put(fib_event->men_info.mfc); 7630 break; 7631 case FIB_EVENT_ENTRY_DEL: 7632 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7633 mr_cache_put(fib_event->men_info.mfc); 7634 break; 7635 case FIB_EVENT_VIF_ADD: 7636 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7637 &fib_event->ven_info); 7638 if (err) 7639 mlxsw_sp_router_fib_abort(mlxsw_sp); 7640 dev_put(fib_event->ven_info.dev); 7641 break; 7642 case FIB_EVENT_VIF_DEL: 7643 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7644 dev_put(fib_event->ven_info.dev); 7645 break; 7646 } 7647 mutex_unlock(&mlxsw_sp->router->lock); 7648 rtnl_unlock(); 7649 } 7650 7651 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7652 { 7653 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7654 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7655 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7656 struct mlxsw_sp_fib_event *next_fib_event; 7657 struct mlxsw_sp_fib_event *fib_event; 7658 int last_family = AF_UNSPEC; 7659 LIST_HEAD(fib_event_queue); 7660 7661 spin_lock_bh(&router->fib_event_queue_lock); 7662 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7663 spin_unlock_bh(&router->fib_event_queue_lock); 7664 7665 /* Router lock is held here to make sure per-instance 7666 * operation context is not used in between FIB4/6 events 7667 * processing. 7668 */ 7669 mutex_lock(&router->lock); 7670 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7671 list_for_each_entry_safe(fib_event, next_fib_event, 7672 &fib_event_queue, list) { 7673 /* Check if the next entry in the queue exists and it is 7674 * of the same type (family and event) as the currect one. 7675 * In that case it is permitted to do the bulking 7676 * of multiple FIB entries to a single register write. 7677 */ 7678 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7679 fib_event->family == next_fib_event->family && 7680 fib_event->event == next_fib_event->event; 7681 op_ctx->event = fib_event->event; 7682 7683 /* In case family of this and the previous entry are different, context 7684 * reinitialization is going to be needed now, indicate that. 7685 * Note that since last_family is initialized to AF_UNSPEC, this is always 7686 * going to happen for the first entry processed in the work. 7687 */ 7688 if (fib_event->family != last_family) 7689 op_ctx->initialized = false; 7690 7691 switch (fib_event->family) { 7692 case AF_INET: 7693 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7694 fib_event); 7695 break; 7696 case AF_INET6: 7697 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7698 fib_event); 7699 break; 7700 case RTNL_FAMILY_IP6MR: 7701 case RTNL_FAMILY_IPMR: 7702 /* Unlock here as inside FIBMR the lock is taken again 7703 * under RTNL. The per-instance operation context 7704 * is not used by FIBMR. 7705 */ 7706 mutex_unlock(&router->lock); 7707 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7708 fib_event); 7709 mutex_lock(&router->lock); 7710 break; 7711 default: 7712 WARN_ON_ONCE(1); 7713 } 7714 last_family = fib_event->family; 7715 kfree(fib_event); 7716 cond_resched(); 7717 } 7718 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7719 mutex_unlock(&router->lock); 7720 } 7721 7722 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7723 struct fib_notifier_info *info) 7724 { 7725 struct fib_entry_notifier_info *fen_info; 7726 struct fib_nh_notifier_info *fnh_info; 7727 7728 switch (fib_event->event) { 7729 case FIB_EVENT_ENTRY_REPLACE: 7730 case FIB_EVENT_ENTRY_DEL: 7731 fen_info = container_of(info, struct fib_entry_notifier_info, 7732 info); 7733 fib_event->fen_info = *fen_info; 7734 /* Take reference on fib_info to prevent it from being 7735 * freed while event is queued. Release it afterwards. 7736 */ 7737 fib_info_hold(fib_event->fen_info.fi); 7738 break; 7739 case FIB_EVENT_NH_ADD: 7740 case FIB_EVENT_NH_DEL: 7741 fnh_info = container_of(info, struct fib_nh_notifier_info, 7742 info); 7743 fib_event->fnh_info = *fnh_info; 7744 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7745 break; 7746 } 7747 } 7748 7749 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7750 struct fib_notifier_info *info) 7751 { 7752 struct fib6_entry_notifier_info *fen6_info; 7753 int err; 7754 7755 switch (fib_event->event) { 7756 case FIB_EVENT_ENTRY_REPLACE: 7757 case FIB_EVENT_ENTRY_APPEND: 7758 case FIB_EVENT_ENTRY_DEL: 7759 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7760 info); 7761 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7762 fen6_info); 7763 if (err) 7764 return err; 7765 break; 7766 } 7767 7768 return 0; 7769 } 7770 7771 static void 7772 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7773 struct fib_notifier_info *info) 7774 { 7775 switch (fib_event->event) { 7776 case FIB_EVENT_ENTRY_REPLACE: 7777 case FIB_EVENT_ENTRY_ADD: 7778 case FIB_EVENT_ENTRY_DEL: 7779 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7780 mr_cache_hold(fib_event->men_info.mfc); 7781 break; 7782 case FIB_EVENT_VIF_ADD: 7783 case FIB_EVENT_VIF_DEL: 7784 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7785 dev_hold(fib_event->ven_info.dev); 7786 break; 7787 } 7788 } 7789 7790 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7791 struct fib_notifier_info *info, 7792 struct mlxsw_sp *mlxsw_sp) 7793 { 7794 struct netlink_ext_ack *extack = info->extack; 7795 struct fib_rule_notifier_info *fr_info; 7796 struct fib_rule *rule; 7797 int err = 0; 7798 7799 /* nothing to do at the moment */ 7800 if (event == FIB_EVENT_RULE_DEL) 7801 return 0; 7802 7803 if (mlxsw_sp->router->aborted) 7804 return 0; 7805 7806 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7807 rule = fr_info->rule; 7808 7809 /* Rule only affects locally generated traffic */ 7810 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7811 return 0; 7812 7813 switch (info->family) { 7814 case AF_INET: 7815 if (!fib4_rule_default(rule) && !rule->l3mdev) 7816 err = -EOPNOTSUPP; 7817 break; 7818 case AF_INET6: 7819 if (!fib6_rule_default(rule) && !rule->l3mdev) 7820 err = -EOPNOTSUPP; 7821 break; 7822 case RTNL_FAMILY_IPMR: 7823 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7824 err = -EOPNOTSUPP; 7825 break; 7826 case RTNL_FAMILY_IP6MR: 7827 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7828 err = -EOPNOTSUPP; 7829 break; 7830 } 7831 7832 if (err < 0) 7833 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7834 7835 return err; 7836 } 7837 7838 /* Called with rcu_read_lock() */ 7839 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7840 unsigned long event, void *ptr) 7841 { 7842 struct mlxsw_sp_fib_event *fib_event; 7843 struct fib_notifier_info *info = ptr; 7844 struct mlxsw_sp_router *router; 7845 int err; 7846 7847 if ((info->family != AF_INET && info->family != AF_INET6 && 7848 info->family != RTNL_FAMILY_IPMR && 7849 info->family != RTNL_FAMILY_IP6MR)) 7850 return NOTIFY_DONE; 7851 7852 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7853 7854 switch (event) { 7855 case FIB_EVENT_RULE_ADD: 7856 case FIB_EVENT_RULE_DEL: 7857 err = mlxsw_sp_router_fib_rule_event(event, info, 7858 router->mlxsw_sp); 7859 return notifier_from_errno(err); 7860 case FIB_EVENT_ENTRY_ADD: 7861 case FIB_EVENT_ENTRY_REPLACE: 7862 case FIB_EVENT_ENTRY_APPEND: 7863 if (router->aborted) { 7864 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route"); 7865 return notifier_from_errno(-EINVAL); 7866 } 7867 if (info->family == AF_INET) { 7868 struct fib_entry_notifier_info *fen_info = ptr; 7869 7870 if (fen_info->fi->fib_nh_is_v6) { 7871 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7872 return notifier_from_errno(-EINVAL); 7873 } 7874 } 7875 break; 7876 } 7877 7878 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7879 if (!fib_event) 7880 return NOTIFY_BAD; 7881 7882 fib_event->mlxsw_sp = router->mlxsw_sp; 7883 fib_event->event = event; 7884 fib_event->family = info->family; 7885 7886 switch (info->family) { 7887 case AF_INET: 7888 mlxsw_sp_router_fib4_event(fib_event, info); 7889 break; 7890 case AF_INET6: 7891 err = mlxsw_sp_router_fib6_event(fib_event, info); 7892 if (err) 7893 goto err_fib_event; 7894 break; 7895 case RTNL_FAMILY_IP6MR: 7896 case RTNL_FAMILY_IPMR: 7897 mlxsw_sp_router_fibmr_event(fib_event, info); 7898 break; 7899 } 7900 7901 /* Enqueue the event and trigger the work */ 7902 spin_lock_bh(&router->fib_event_queue_lock); 7903 list_add_tail(&fib_event->list, &router->fib_event_queue); 7904 spin_unlock_bh(&router->fib_event_queue_lock); 7905 mlxsw_core_schedule_work(&router->fib_event_work); 7906 7907 return NOTIFY_DONE; 7908 7909 err_fib_event: 7910 kfree(fib_event); 7911 return NOTIFY_BAD; 7912 } 7913 7914 static struct mlxsw_sp_rif * 7915 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7916 const struct net_device *dev) 7917 { 7918 int i; 7919 7920 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7921 if (mlxsw_sp->router->rifs[i] && 7922 mlxsw_sp->router->rifs[i]->dev == dev) 7923 return mlxsw_sp->router->rifs[i]; 7924 7925 return NULL; 7926 } 7927 7928 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7929 const struct net_device *dev) 7930 { 7931 struct mlxsw_sp_rif *rif; 7932 7933 mutex_lock(&mlxsw_sp->router->lock); 7934 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7935 mutex_unlock(&mlxsw_sp->router->lock); 7936 7937 return rif; 7938 } 7939 7940 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7941 { 7942 struct mlxsw_sp_rif *rif; 7943 u16 vid = 0; 7944 7945 mutex_lock(&mlxsw_sp->router->lock); 7946 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7947 if (!rif) 7948 goto out; 7949 7950 /* We only return the VID for VLAN RIFs. Otherwise we return an 7951 * invalid value (0). 7952 */ 7953 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7954 goto out; 7955 7956 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7957 7958 out: 7959 mutex_unlock(&mlxsw_sp->router->lock); 7960 return vid; 7961 } 7962 7963 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7964 { 7965 char ritr_pl[MLXSW_REG_RITR_LEN]; 7966 int err; 7967 7968 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7969 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7970 if (err) 7971 return err; 7972 7973 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7974 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7975 } 7976 7977 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7978 struct mlxsw_sp_rif *rif) 7979 { 7980 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7981 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7982 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7983 } 7984 7985 static bool 7986 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7987 unsigned long event) 7988 { 7989 struct inet6_dev *inet6_dev; 7990 bool addr_list_empty = true; 7991 struct in_device *idev; 7992 7993 switch (event) { 7994 case NETDEV_UP: 7995 return rif == NULL; 7996 case NETDEV_DOWN: 7997 rcu_read_lock(); 7998 idev = __in_dev_get_rcu(dev); 7999 if (idev && idev->ifa_list) 8000 addr_list_empty = false; 8001 8002 inet6_dev = __in6_dev_get(dev); 8003 if (addr_list_empty && inet6_dev && 8004 !list_empty(&inet6_dev->addr_list)) 8005 addr_list_empty = false; 8006 rcu_read_unlock(); 8007 8008 /* macvlans do not have a RIF, but rather piggy back on the 8009 * RIF of their lower device. 8010 */ 8011 if (netif_is_macvlan(dev) && addr_list_empty) 8012 return true; 8013 8014 if (rif && addr_list_empty && 8015 !netif_is_l3_slave(rif->dev)) 8016 return true; 8017 /* It is possible we already removed the RIF ourselves 8018 * if it was assigned to a netdev that is now a bridge 8019 * or LAG slave. 8020 */ 8021 return false; 8022 } 8023 8024 return false; 8025 } 8026 8027 static enum mlxsw_sp_rif_type 8028 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8029 const struct net_device *dev) 8030 { 8031 enum mlxsw_sp_fid_type type; 8032 8033 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8034 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8035 8036 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8037 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8038 type = MLXSW_SP_FID_TYPE_8021Q; 8039 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8040 type = MLXSW_SP_FID_TYPE_8021Q; 8041 else if (netif_is_bridge_master(dev)) 8042 type = MLXSW_SP_FID_TYPE_8021D; 8043 else 8044 type = MLXSW_SP_FID_TYPE_RFID; 8045 8046 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8047 } 8048 8049 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 8050 { 8051 int i; 8052 8053 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8054 if (!mlxsw_sp->router->rifs[i]) { 8055 *p_rif_index = i; 8056 return 0; 8057 } 8058 } 8059 8060 return -ENOBUFS; 8061 } 8062 8063 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8064 u16 vr_id, 8065 struct net_device *l3_dev) 8066 { 8067 struct mlxsw_sp_rif *rif; 8068 8069 rif = kzalloc(rif_size, GFP_KERNEL); 8070 if (!rif) 8071 return NULL; 8072 8073 INIT_LIST_HEAD(&rif->nexthop_list); 8074 INIT_LIST_HEAD(&rif->neigh_list); 8075 if (l3_dev) { 8076 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8077 rif->mtu = l3_dev->mtu; 8078 rif->dev = l3_dev; 8079 } 8080 rif->vr_id = vr_id; 8081 rif->rif_index = rif_index; 8082 8083 return rif; 8084 } 8085 8086 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8087 u16 rif_index) 8088 { 8089 return mlxsw_sp->router->rifs[rif_index]; 8090 } 8091 8092 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8093 { 8094 return rif->rif_index; 8095 } 8096 8097 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8098 { 8099 return lb_rif->common.rif_index; 8100 } 8101 8102 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8103 { 8104 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 8105 struct mlxsw_sp_vr *ul_vr; 8106 8107 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8108 if (WARN_ON(IS_ERR(ul_vr))) 8109 return 0; 8110 8111 return ul_vr->id; 8112 } 8113 8114 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8115 { 8116 return lb_rif->ul_rif_id; 8117 } 8118 8119 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8120 { 8121 return rif->dev->ifindex; 8122 } 8123 8124 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 8125 { 8126 return rif->dev; 8127 } 8128 8129 static struct mlxsw_sp_rif * 8130 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8131 const struct mlxsw_sp_rif_params *params, 8132 struct netlink_ext_ack *extack) 8133 { 8134 u32 tb_id = l3mdev_fib_table(params->dev); 8135 const struct mlxsw_sp_rif_ops *ops; 8136 struct mlxsw_sp_fid *fid = NULL; 8137 enum mlxsw_sp_rif_type type; 8138 struct mlxsw_sp_rif *rif; 8139 struct mlxsw_sp_vr *vr; 8140 u16 rif_index; 8141 int i, err; 8142 8143 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8144 ops = mlxsw_sp->router->rif_ops_arr[type]; 8145 8146 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8147 if (IS_ERR(vr)) 8148 return ERR_CAST(vr); 8149 vr->rif_count++; 8150 8151 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8152 if (err) { 8153 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8154 goto err_rif_index_alloc; 8155 } 8156 8157 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 8158 if (!rif) { 8159 err = -ENOMEM; 8160 goto err_rif_alloc; 8161 } 8162 dev_hold(rif->dev); 8163 mlxsw_sp->router->rifs[rif_index] = rif; 8164 rif->mlxsw_sp = mlxsw_sp; 8165 rif->ops = ops; 8166 8167 if (ops->fid_get) { 8168 fid = ops->fid_get(rif, extack); 8169 if (IS_ERR(fid)) { 8170 err = PTR_ERR(fid); 8171 goto err_fid_get; 8172 } 8173 rif->fid = fid; 8174 } 8175 8176 if (ops->setup) 8177 ops->setup(rif, params); 8178 8179 err = ops->configure(rif); 8180 if (err) 8181 goto err_configure; 8182 8183 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8184 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8185 if (err) 8186 goto err_mr_rif_add; 8187 } 8188 8189 mlxsw_sp_rif_counters_alloc(rif); 8190 8191 return rif; 8192 8193 err_mr_rif_add: 8194 for (i--; i >= 0; i--) 8195 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8196 ops->deconfigure(rif); 8197 err_configure: 8198 if (fid) 8199 mlxsw_sp_fid_put(fid); 8200 err_fid_get: 8201 mlxsw_sp->router->rifs[rif_index] = NULL; 8202 dev_put(rif->dev); 8203 kfree(rif); 8204 err_rif_alloc: 8205 err_rif_index_alloc: 8206 vr->rif_count--; 8207 mlxsw_sp_vr_put(mlxsw_sp, vr); 8208 return ERR_PTR(err); 8209 } 8210 8211 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8212 { 8213 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8214 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8215 struct mlxsw_sp_fid *fid = rif->fid; 8216 struct mlxsw_sp_vr *vr; 8217 int i; 8218 8219 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8220 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8221 8222 mlxsw_sp_rif_counters_free(rif); 8223 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8224 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8225 ops->deconfigure(rif); 8226 if (fid) 8227 /* Loopback RIFs are not associated with a FID. */ 8228 mlxsw_sp_fid_put(fid); 8229 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8230 dev_put(rif->dev); 8231 kfree(rif); 8232 vr->rif_count--; 8233 mlxsw_sp_vr_put(mlxsw_sp, vr); 8234 } 8235 8236 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8237 struct net_device *dev) 8238 { 8239 struct mlxsw_sp_rif *rif; 8240 8241 mutex_lock(&mlxsw_sp->router->lock); 8242 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8243 if (!rif) 8244 goto out; 8245 mlxsw_sp_rif_destroy(rif); 8246 out: 8247 mutex_unlock(&mlxsw_sp->router->lock); 8248 } 8249 8250 static void 8251 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8252 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8253 { 8254 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8255 8256 params->vid = mlxsw_sp_port_vlan->vid; 8257 params->lag = mlxsw_sp_port->lagged; 8258 if (params->lag) 8259 params->lag_id = mlxsw_sp_port->lag_id; 8260 else 8261 params->system_port = mlxsw_sp_port->local_port; 8262 } 8263 8264 static struct mlxsw_sp_rif_subport * 8265 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8266 { 8267 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8268 } 8269 8270 static struct mlxsw_sp_rif * 8271 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8272 const struct mlxsw_sp_rif_params *params, 8273 struct netlink_ext_ack *extack) 8274 { 8275 struct mlxsw_sp_rif_subport *rif_subport; 8276 struct mlxsw_sp_rif *rif; 8277 8278 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8279 if (!rif) 8280 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8281 8282 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8283 refcount_inc(&rif_subport->ref_count); 8284 return rif; 8285 } 8286 8287 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8288 { 8289 struct mlxsw_sp_rif_subport *rif_subport; 8290 8291 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8292 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8293 return; 8294 8295 mlxsw_sp_rif_destroy(rif); 8296 } 8297 8298 static int 8299 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8300 struct net_device *l3_dev, 8301 struct netlink_ext_ack *extack) 8302 { 8303 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8304 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8305 struct mlxsw_sp_rif_params params = { 8306 .dev = l3_dev, 8307 }; 8308 u16 vid = mlxsw_sp_port_vlan->vid; 8309 struct mlxsw_sp_rif *rif; 8310 struct mlxsw_sp_fid *fid; 8311 int err; 8312 8313 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8314 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8315 if (IS_ERR(rif)) 8316 return PTR_ERR(rif); 8317 8318 /* FID was already created, just take a reference */ 8319 fid = rif->ops->fid_get(rif, extack); 8320 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8321 if (err) 8322 goto err_fid_port_vid_map; 8323 8324 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8325 if (err) 8326 goto err_port_vid_learning_set; 8327 8328 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8329 BR_STATE_FORWARDING); 8330 if (err) 8331 goto err_port_vid_stp_set; 8332 8333 mlxsw_sp_port_vlan->fid = fid; 8334 8335 return 0; 8336 8337 err_port_vid_stp_set: 8338 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8339 err_port_vid_learning_set: 8340 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8341 err_fid_port_vid_map: 8342 mlxsw_sp_fid_put(fid); 8343 mlxsw_sp_rif_subport_put(rif); 8344 return err; 8345 } 8346 8347 static void 8348 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8349 { 8350 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8351 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8352 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8353 u16 vid = mlxsw_sp_port_vlan->vid; 8354 8355 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8356 return; 8357 8358 mlxsw_sp_port_vlan->fid = NULL; 8359 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8360 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8361 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8362 mlxsw_sp_fid_put(fid); 8363 mlxsw_sp_rif_subport_put(rif); 8364 } 8365 8366 int 8367 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8368 struct net_device *l3_dev, 8369 struct netlink_ext_ack *extack) 8370 { 8371 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8372 struct mlxsw_sp_rif *rif; 8373 int err = 0; 8374 8375 mutex_lock(&mlxsw_sp->router->lock); 8376 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8377 if (!rif) 8378 goto out; 8379 8380 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8381 extack); 8382 out: 8383 mutex_unlock(&mlxsw_sp->router->lock); 8384 return err; 8385 } 8386 8387 void 8388 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8389 { 8390 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8391 8392 mutex_lock(&mlxsw_sp->router->lock); 8393 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8394 mutex_unlock(&mlxsw_sp->router->lock); 8395 } 8396 8397 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8398 struct net_device *port_dev, 8399 unsigned long event, u16 vid, 8400 struct netlink_ext_ack *extack) 8401 { 8402 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8403 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8404 8405 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8406 if (WARN_ON(!mlxsw_sp_port_vlan)) 8407 return -EINVAL; 8408 8409 switch (event) { 8410 case NETDEV_UP: 8411 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8412 l3_dev, extack); 8413 case NETDEV_DOWN: 8414 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8415 break; 8416 } 8417 8418 return 0; 8419 } 8420 8421 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8422 unsigned long event, 8423 struct netlink_ext_ack *extack) 8424 { 8425 if (netif_is_bridge_port(port_dev) || 8426 netif_is_lag_port(port_dev) || 8427 netif_is_ovs_port(port_dev)) 8428 return 0; 8429 8430 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8431 MLXSW_SP_DEFAULT_VID, extack); 8432 } 8433 8434 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8435 struct net_device *lag_dev, 8436 unsigned long event, u16 vid, 8437 struct netlink_ext_ack *extack) 8438 { 8439 struct net_device *port_dev; 8440 struct list_head *iter; 8441 int err; 8442 8443 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8444 if (mlxsw_sp_port_dev_check(port_dev)) { 8445 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8446 port_dev, 8447 event, vid, 8448 extack); 8449 if (err) 8450 return err; 8451 } 8452 } 8453 8454 return 0; 8455 } 8456 8457 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8458 unsigned long event, 8459 struct netlink_ext_ack *extack) 8460 { 8461 if (netif_is_bridge_port(lag_dev)) 8462 return 0; 8463 8464 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8465 MLXSW_SP_DEFAULT_VID, extack); 8466 } 8467 8468 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8469 struct net_device *l3_dev, 8470 unsigned long event, 8471 struct netlink_ext_ack *extack) 8472 { 8473 struct mlxsw_sp_rif_params params = { 8474 .dev = l3_dev, 8475 }; 8476 struct mlxsw_sp_rif *rif; 8477 8478 switch (event) { 8479 case NETDEV_UP: 8480 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8481 u16 proto; 8482 8483 br_vlan_get_proto(l3_dev, &proto); 8484 if (proto == ETH_P_8021AD) { 8485 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8486 return -EOPNOTSUPP; 8487 } 8488 } 8489 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8490 if (IS_ERR(rif)) 8491 return PTR_ERR(rif); 8492 break; 8493 case NETDEV_DOWN: 8494 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8495 mlxsw_sp_rif_destroy(rif); 8496 break; 8497 } 8498 8499 return 0; 8500 } 8501 8502 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8503 struct net_device *vlan_dev, 8504 unsigned long event, 8505 struct netlink_ext_ack *extack) 8506 { 8507 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8508 u16 vid = vlan_dev_vlan_id(vlan_dev); 8509 8510 if (netif_is_bridge_port(vlan_dev)) 8511 return 0; 8512 8513 if (mlxsw_sp_port_dev_check(real_dev)) 8514 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8515 event, vid, extack); 8516 else if (netif_is_lag_master(real_dev)) 8517 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8518 vid, extack); 8519 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8520 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8521 extack); 8522 8523 return 0; 8524 } 8525 8526 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8527 { 8528 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8529 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8530 8531 return ether_addr_equal_masked(mac, vrrp4, mask); 8532 } 8533 8534 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8535 { 8536 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8537 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8538 8539 return ether_addr_equal_masked(mac, vrrp6, mask); 8540 } 8541 8542 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8543 const u8 *mac, bool adding) 8544 { 8545 char ritr_pl[MLXSW_REG_RITR_LEN]; 8546 u8 vrrp_id = adding ? mac[5] : 0; 8547 int err; 8548 8549 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8550 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8551 return 0; 8552 8553 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8554 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8555 if (err) 8556 return err; 8557 8558 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8559 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8560 else 8561 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8562 8563 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8564 } 8565 8566 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8567 const struct net_device *macvlan_dev, 8568 struct netlink_ext_ack *extack) 8569 { 8570 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8571 struct mlxsw_sp_rif *rif; 8572 int err; 8573 8574 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8575 if (!rif) { 8576 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8577 return -EOPNOTSUPP; 8578 } 8579 8580 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8581 mlxsw_sp_fid_index(rif->fid), true); 8582 if (err) 8583 return err; 8584 8585 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8586 macvlan_dev->dev_addr, true); 8587 if (err) 8588 goto err_rif_vrrp_add; 8589 8590 /* Make sure the bridge driver does not have this MAC pointing at 8591 * some other port. 8592 */ 8593 if (rif->ops->fdb_del) 8594 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8595 8596 return 0; 8597 8598 err_rif_vrrp_add: 8599 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8600 mlxsw_sp_fid_index(rif->fid), false); 8601 return err; 8602 } 8603 8604 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8605 const struct net_device *macvlan_dev) 8606 { 8607 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8608 struct mlxsw_sp_rif *rif; 8609 8610 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8611 /* If we do not have a RIF, then we already took care of 8612 * removing the macvlan's MAC during RIF deletion. 8613 */ 8614 if (!rif) 8615 return; 8616 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8617 false); 8618 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8619 mlxsw_sp_fid_index(rif->fid), false); 8620 } 8621 8622 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8623 const struct net_device *macvlan_dev) 8624 { 8625 mutex_lock(&mlxsw_sp->router->lock); 8626 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8627 mutex_unlock(&mlxsw_sp->router->lock); 8628 } 8629 8630 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8631 struct net_device *macvlan_dev, 8632 unsigned long event, 8633 struct netlink_ext_ack *extack) 8634 { 8635 switch (event) { 8636 case NETDEV_UP: 8637 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8638 case NETDEV_DOWN: 8639 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8640 break; 8641 } 8642 8643 return 0; 8644 } 8645 8646 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp, 8647 struct net_device *dev, 8648 const unsigned char *dev_addr, 8649 struct netlink_ext_ack *extack) 8650 { 8651 struct mlxsw_sp_rif *rif; 8652 int i; 8653 8654 /* A RIF is not created for macvlan netdevs. Their MAC is used to 8655 * populate the FDB 8656 */ 8657 if (netif_is_macvlan(dev) || netif_is_l3_master(dev)) 8658 return 0; 8659 8660 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8661 rif = mlxsw_sp->router->rifs[i]; 8662 if (rif && rif->ops && 8663 rif->ops->type == MLXSW_SP_RIF_TYPE_IPIP_LB) 8664 continue; 8665 if (rif && rif->dev && rif->dev != dev && 8666 !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr, 8667 mlxsw_sp->mac_mask)) { 8668 NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix"); 8669 return -EINVAL; 8670 } 8671 } 8672 8673 return 0; 8674 } 8675 8676 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8677 struct net_device *dev, 8678 unsigned long event, 8679 struct netlink_ext_ack *extack) 8680 { 8681 if (mlxsw_sp_port_dev_check(dev)) 8682 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8683 else if (netif_is_lag_master(dev)) 8684 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8685 else if (netif_is_bridge_master(dev)) 8686 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8687 extack); 8688 else if (is_vlan_dev(dev)) 8689 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8690 extack); 8691 else if (netif_is_macvlan(dev)) 8692 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8693 extack); 8694 else 8695 return 0; 8696 } 8697 8698 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8699 unsigned long event, void *ptr) 8700 { 8701 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8702 struct net_device *dev = ifa->ifa_dev->dev; 8703 struct mlxsw_sp_router *router; 8704 struct mlxsw_sp_rif *rif; 8705 int err = 0; 8706 8707 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8708 if (event == NETDEV_UP) 8709 return NOTIFY_DONE; 8710 8711 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8712 mutex_lock(&router->lock); 8713 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8714 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8715 goto out; 8716 8717 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8718 out: 8719 mutex_unlock(&router->lock); 8720 return notifier_from_errno(err); 8721 } 8722 8723 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8724 unsigned long event, void *ptr) 8725 { 8726 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8727 struct net_device *dev = ivi->ivi_dev->dev; 8728 struct mlxsw_sp *mlxsw_sp; 8729 struct mlxsw_sp_rif *rif; 8730 int err = 0; 8731 8732 mlxsw_sp = mlxsw_sp_lower_get(dev); 8733 if (!mlxsw_sp) 8734 return NOTIFY_DONE; 8735 8736 mutex_lock(&mlxsw_sp->router->lock); 8737 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8738 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8739 goto out; 8740 8741 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8742 ivi->extack); 8743 if (err) 8744 goto out; 8745 8746 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8747 out: 8748 mutex_unlock(&mlxsw_sp->router->lock); 8749 return notifier_from_errno(err); 8750 } 8751 8752 struct mlxsw_sp_inet6addr_event_work { 8753 struct work_struct work; 8754 struct mlxsw_sp *mlxsw_sp; 8755 struct net_device *dev; 8756 unsigned long event; 8757 }; 8758 8759 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8760 { 8761 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8762 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8763 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8764 struct net_device *dev = inet6addr_work->dev; 8765 unsigned long event = inet6addr_work->event; 8766 struct mlxsw_sp_rif *rif; 8767 8768 rtnl_lock(); 8769 mutex_lock(&mlxsw_sp->router->lock); 8770 8771 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8772 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8773 goto out; 8774 8775 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8776 out: 8777 mutex_unlock(&mlxsw_sp->router->lock); 8778 rtnl_unlock(); 8779 dev_put(dev); 8780 kfree(inet6addr_work); 8781 } 8782 8783 /* Called with rcu_read_lock() */ 8784 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8785 unsigned long event, void *ptr) 8786 { 8787 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8788 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8789 struct net_device *dev = if6->idev->dev; 8790 struct mlxsw_sp_router *router; 8791 8792 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8793 if (event == NETDEV_UP) 8794 return NOTIFY_DONE; 8795 8796 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8797 if (!inet6addr_work) 8798 return NOTIFY_BAD; 8799 8800 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8801 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8802 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8803 inet6addr_work->dev = dev; 8804 inet6addr_work->event = event; 8805 dev_hold(dev); 8806 mlxsw_core_schedule_work(&inet6addr_work->work); 8807 8808 return NOTIFY_DONE; 8809 } 8810 8811 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8812 unsigned long event, void *ptr) 8813 { 8814 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8815 struct net_device *dev = i6vi->i6vi_dev->dev; 8816 struct mlxsw_sp *mlxsw_sp; 8817 struct mlxsw_sp_rif *rif; 8818 int err = 0; 8819 8820 mlxsw_sp = mlxsw_sp_lower_get(dev); 8821 if (!mlxsw_sp) 8822 return NOTIFY_DONE; 8823 8824 mutex_lock(&mlxsw_sp->router->lock); 8825 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8826 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8827 goto out; 8828 8829 err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr, 8830 i6vi->extack); 8831 if (err) 8832 goto out; 8833 8834 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8835 out: 8836 mutex_unlock(&mlxsw_sp->router->lock); 8837 return notifier_from_errno(err); 8838 } 8839 8840 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8841 const char *mac, int mtu) 8842 { 8843 char ritr_pl[MLXSW_REG_RITR_LEN]; 8844 int err; 8845 8846 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8847 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8848 if (err) 8849 return err; 8850 8851 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8852 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8853 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8854 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8855 } 8856 8857 static int 8858 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8859 struct mlxsw_sp_rif *rif) 8860 { 8861 struct net_device *dev = rif->dev; 8862 u16 fid_index; 8863 int err; 8864 8865 fid_index = mlxsw_sp_fid_index(rif->fid); 8866 8867 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8868 if (err) 8869 return err; 8870 8871 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8872 dev->mtu); 8873 if (err) 8874 goto err_rif_edit; 8875 8876 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 8877 if (err) 8878 goto err_rif_fdb_op; 8879 8880 if (rif->mtu != dev->mtu) { 8881 struct mlxsw_sp_vr *vr; 8882 int i; 8883 8884 /* The RIF is relevant only to its mr_table instance, as unlike 8885 * unicast routing, in multicast routing a RIF cannot be shared 8886 * between several multicast routing tables. 8887 */ 8888 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8889 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8890 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 8891 rif, dev->mtu); 8892 } 8893 8894 ether_addr_copy(rif->addr, dev->dev_addr); 8895 rif->mtu = dev->mtu; 8896 8897 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 8898 8899 return 0; 8900 8901 err_rif_fdb_op: 8902 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu); 8903 err_rif_edit: 8904 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 8905 return err; 8906 } 8907 8908 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 8909 struct netdev_notifier_pre_changeaddr_info *info) 8910 { 8911 struct netlink_ext_ack *extack; 8912 8913 extack = netdev_notifier_info_to_extack(&info->info); 8914 return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev, 8915 info->dev_addr, extack); 8916 } 8917 8918 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 8919 unsigned long event, void *ptr) 8920 { 8921 struct mlxsw_sp *mlxsw_sp; 8922 struct mlxsw_sp_rif *rif; 8923 int err = 0; 8924 8925 mlxsw_sp = mlxsw_sp_lower_get(dev); 8926 if (!mlxsw_sp) 8927 return 0; 8928 8929 mutex_lock(&mlxsw_sp->router->lock); 8930 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8931 if (!rif) 8932 goto out; 8933 8934 switch (event) { 8935 case NETDEV_CHANGEMTU: 8936 case NETDEV_CHANGEADDR: 8937 err = mlxsw_sp_router_port_change_event(mlxsw_sp, rif); 8938 break; 8939 case NETDEV_PRE_CHANGEADDR: 8940 err = mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 8941 break; 8942 } 8943 8944 out: 8945 mutex_unlock(&mlxsw_sp->router->lock); 8946 return err; 8947 } 8948 8949 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 8950 struct net_device *l3_dev, 8951 struct netlink_ext_ack *extack) 8952 { 8953 struct mlxsw_sp_rif *rif; 8954 8955 /* If netdev is already associated with a RIF, then we need to 8956 * destroy it and create a new one with the new virtual router ID. 8957 */ 8958 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8959 if (rif) 8960 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 8961 extack); 8962 8963 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 8964 } 8965 8966 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 8967 struct net_device *l3_dev) 8968 { 8969 struct mlxsw_sp_rif *rif; 8970 8971 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8972 if (!rif) 8973 return; 8974 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 8975 } 8976 8977 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 8978 struct netdev_notifier_changeupper_info *info) 8979 { 8980 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 8981 int err = 0; 8982 8983 /* We do not create a RIF for a macvlan, but only use it to 8984 * direct more MAC addresses to the router. 8985 */ 8986 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 8987 return 0; 8988 8989 mutex_lock(&mlxsw_sp->router->lock); 8990 switch (event) { 8991 case NETDEV_PRECHANGEUPPER: 8992 break; 8993 case NETDEV_CHANGEUPPER: 8994 if (info->linking) { 8995 struct netlink_ext_ack *extack; 8996 8997 extack = netdev_notifier_info_to_extack(&info->info); 8998 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 8999 } else { 9000 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9001 } 9002 break; 9003 } 9004 mutex_unlock(&mlxsw_sp->router->lock); 9005 9006 return err; 9007 } 9008 9009 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 9010 struct netdev_nested_priv *priv) 9011 { 9012 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 9013 9014 if (!netif_is_macvlan(dev)) 9015 return 0; 9016 9017 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9018 mlxsw_sp_fid_index(rif->fid), false); 9019 } 9020 9021 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 9022 { 9023 struct netdev_nested_priv priv = { 9024 .data = (void *)rif, 9025 }; 9026 9027 if (!netif_is_macvlan_port(rif->dev)) 9028 return 0; 9029 9030 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 9031 return netdev_walk_all_upper_dev_rcu(rif->dev, 9032 __mlxsw_sp_rif_macvlan_flush, &priv); 9033 } 9034 9035 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 9036 const struct mlxsw_sp_rif_params *params) 9037 { 9038 struct mlxsw_sp_rif_subport *rif_subport; 9039 9040 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9041 refcount_set(&rif_subport->ref_count, 1); 9042 rif_subport->vid = params->vid; 9043 rif_subport->lag = params->lag; 9044 if (params->lag) 9045 rif_subport->lag_id = params->lag_id; 9046 else 9047 rif_subport->system_port = params->system_port; 9048 } 9049 9050 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 9051 { 9052 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9053 struct mlxsw_sp_rif_subport *rif_subport; 9054 char ritr_pl[MLXSW_REG_RITR_LEN]; 9055 9056 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9057 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 9058 rif->rif_index, rif->vr_id, rif->dev->mtu); 9059 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9060 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 9061 rif_subport->lag ? rif_subport->lag_id : 9062 rif_subport->system_port, 9063 rif_subport->vid); 9064 9065 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9066 } 9067 9068 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif) 9069 { 9070 int err; 9071 9072 err = mlxsw_sp_rif_subport_op(rif, true); 9073 if (err) 9074 return err; 9075 9076 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9077 mlxsw_sp_fid_index(rif->fid), true); 9078 if (err) 9079 goto err_rif_fdb_op; 9080 9081 mlxsw_sp_fid_rif_set(rif->fid, rif); 9082 return 0; 9083 9084 err_rif_fdb_op: 9085 mlxsw_sp_rif_subport_op(rif, false); 9086 return err; 9087 } 9088 9089 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 9090 { 9091 struct mlxsw_sp_fid *fid = rif->fid; 9092 9093 mlxsw_sp_fid_rif_set(fid, NULL); 9094 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9095 mlxsw_sp_fid_index(fid), false); 9096 mlxsw_sp_rif_macvlan_flush(rif); 9097 mlxsw_sp_rif_subport_op(rif, false); 9098 } 9099 9100 static struct mlxsw_sp_fid * 9101 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 9102 struct netlink_ext_ack *extack) 9103 { 9104 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 9105 } 9106 9107 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 9108 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 9109 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 9110 .setup = mlxsw_sp_rif_subport_setup, 9111 .configure = mlxsw_sp_rif_subport_configure, 9112 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 9113 .fid_get = mlxsw_sp_rif_subport_fid_get, 9114 }; 9115 9116 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 9117 enum mlxsw_reg_ritr_if_type type, 9118 u16 vid_fid, bool enable) 9119 { 9120 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9121 char ritr_pl[MLXSW_REG_RITR_LEN]; 9122 9123 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9124 rif->dev->mtu); 9125 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9126 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 9127 9128 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9129 } 9130 9131 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9132 { 9133 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9134 } 9135 9136 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif) 9137 { 9138 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9139 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9140 int err; 9141 9142 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 9143 true); 9144 if (err) 9145 return err; 9146 9147 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9148 mlxsw_sp_router_port(mlxsw_sp), true); 9149 if (err) 9150 goto err_fid_mc_flood_set; 9151 9152 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9153 mlxsw_sp_router_port(mlxsw_sp), true); 9154 if (err) 9155 goto err_fid_bc_flood_set; 9156 9157 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9158 mlxsw_sp_fid_index(rif->fid), true); 9159 if (err) 9160 goto err_rif_fdb_op; 9161 9162 mlxsw_sp_fid_rif_set(rif->fid, rif); 9163 return 0; 9164 9165 err_rif_fdb_op: 9166 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9167 mlxsw_sp_router_port(mlxsw_sp), false); 9168 err_fid_bc_flood_set: 9169 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9170 mlxsw_sp_router_port(mlxsw_sp), false); 9171 err_fid_mc_flood_set: 9172 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9173 return err; 9174 } 9175 9176 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9177 { 9178 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9179 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9180 struct mlxsw_sp_fid *fid = rif->fid; 9181 9182 mlxsw_sp_fid_rif_set(fid, NULL); 9183 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9184 mlxsw_sp_fid_index(fid), false); 9185 mlxsw_sp_rif_macvlan_flush(rif); 9186 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9187 mlxsw_sp_router_port(mlxsw_sp), false); 9188 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9189 mlxsw_sp_router_port(mlxsw_sp), false); 9190 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9191 } 9192 9193 static struct mlxsw_sp_fid * 9194 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9195 struct netlink_ext_ack *extack) 9196 { 9197 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 9198 } 9199 9200 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9201 { 9202 struct switchdev_notifier_fdb_info info; 9203 struct net_device *dev; 9204 9205 dev = br_fdb_find_port(rif->dev, mac, 0); 9206 if (!dev) 9207 return; 9208 9209 info.addr = mac; 9210 info.vid = 0; 9211 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9212 NULL); 9213 } 9214 9215 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9216 .type = MLXSW_SP_RIF_TYPE_FID, 9217 .rif_size = sizeof(struct mlxsw_sp_rif), 9218 .configure = mlxsw_sp_rif_fid_configure, 9219 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9220 .fid_get = mlxsw_sp_rif_fid_fid_get, 9221 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9222 }; 9223 9224 static struct mlxsw_sp_fid * 9225 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9226 struct netlink_ext_ack *extack) 9227 { 9228 struct net_device *br_dev; 9229 u16 vid; 9230 int err; 9231 9232 if (is_vlan_dev(rif->dev)) { 9233 vid = vlan_dev_vlan_id(rif->dev); 9234 br_dev = vlan_dev_real_dev(rif->dev); 9235 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9236 return ERR_PTR(-EINVAL); 9237 } else { 9238 err = br_vlan_get_pvid(rif->dev, &vid); 9239 if (err < 0 || !vid) { 9240 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9241 return ERR_PTR(-EINVAL); 9242 } 9243 } 9244 9245 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9246 } 9247 9248 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9249 { 9250 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9251 struct switchdev_notifier_fdb_info info; 9252 struct net_device *br_dev; 9253 struct net_device *dev; 9254 9255 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 9256 dev = br_fdb_find_port(br_dev, mac, vid); 9257 if (!dev) 9258 return; 9259 9260 info.addr = mac; 9261 info.vid = vid; 9262 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9263 NULL); 9264 } 9265 9266 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 9267 .type = MLXSW_SP_RIF_TYPE_VLAN, 9268 .rif_size = sizeof(struct mlxsw_sp_rif), 9269 .configure = mlxsw_sp_rif_fid_configure, 9270 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9271 .fid_get = mlxsw_sp_rif_vlan_fid_get, 9272 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 9273 }; 9274 9275 static struct mlxsw_sp_rif_ipip_lb * 9276 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 9277 { 9278 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 9279 } 9280 9281 static void 9282 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 9283 const struct mlxsw_sp_rif_params *params) 9284 { 9285 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 9286 struct mlxsw_sp_rif_ipip_lb *rif_lb; 9287 9288 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 9289 common); 9290 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 9291 rif_lb->lb_config = params_lb->lb_config; 9292 } 9293 9294 static int 9295 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9296 { 9297 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9298 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9299 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9300 struct mlxsw_sp_vr *ul_vr; 9301 int err; 9302 9303 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 9304 if (IS_ERR(ul_vr)) 9305 return PTR_ERR(ul_vr); 9306 9307 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 9308 if (err) 9309 goto err_loopback_op; 9310 9311 lb_rif->ul_vr_id = ul_vr->id; 9312 lb_rif->ul_rif_id = 0; 9313 ++ul_vr->rif_count; 9314 return 0; 9315 9316 err_loopback_op: 9317 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9318 return err; 9319 } 9320 9321 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9322 { 9323 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9324 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9325 struct mlxsw_sp_vr *ul_vr; 9326 9327 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 9328 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 9329 9330 --ul_vr->rif_count; 9331 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9332 } 9333 9334 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 9335 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9336 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9337 .setup = mlxsw_sp_rif_ipip_lb_setup, 9338 .configure = mlxsw_sp1_rif_ipip_lb_configure, 9339 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 9340 }; 9341 9342 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 9343 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9344 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9345 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9346 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 9347 }; 9348 9349 static int 9350 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 9351 { 9352 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9353 char ritr_pl[MLXSW_REG_RITR_LEN]; 9354 9355 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 9356 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 9357 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 9358 MLXSW_REG_RITR_LOOPBACK_GENERIC); 9359 9360 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9361 } 9362 9363 static struct mlxsw_sp_rif * 9364 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 9365 struct netlink_ext_ack *extack) 9366 { 9367 struct mlxsw_sp_rif *ul_rif; 9368 u16 rif_index; 9369 int err; 9370 9371 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 9372 if (err) { 9373 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 9374 return ERR_PTR(err); 9375 } 9376 9377 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 9378 if (!ul_rif) 9379 return ERR_PTR(-ENOMEM); 9380 9381 mlxsw_sp->router->rifs[rif_index] = ul_rif; 9382 ul_rif->mlxsw_sp = mlxsw_sp; 9383 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 9384 if (err) 9385 goto ul_rif_op_err; 9386 9387 return ul_rif; 9388 9389 ul_rif_op_err: 9390 mlxsw_sp->router->rifs[rif_index] = NULL; 9391 kfree(ul_rif); 9392 return ERR_PTR(err); 9393 } 9394 9395 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9396 { 9397 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9398 9399 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9400 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9401 kfree(ul_rif); 9402 } 9403 9404 static struct mlxsw_sp_rif * 9405 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 9406 struct netlink_ext_ack *extack) 9407 { 9408 struct mlxsw_sp_vr *vr; 9409 int err; 9410 9411 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 9412 if (IS_ERR(vr)) 9413 return ERR_CAST(vr); 9414 9415 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 9416 return vr->ul_rif; 9417 9418 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 9419 if (IS_ERR(vr->ul_rif)) { 9420 err = PTR_ERR(vr->ul_rif); 9421 goto err_ul_rif_create; 9422 } 9423 9424 vr->rif_count++; 9425 refcount_set(&vr->ul_rif_refcnt, 1); 9426 9427 return vr->ul_rif; 9428 9429 err_ul_rif_create: 9430 mlxsw_sp_vr_put(mlxsw_sp, vr); 9431 return ERR_PTR(err); 9432 } 9433 9434 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 9435 { 9436 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9437 struct mlxsw_sp_vr *vr; 9438 9439 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 9440 9441 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 9442 return; 9443 9444 vr->rif_count--; 9445 mlxsw_sp_ul_rif_destroy(ul_rif); 9446 mlxsw_sp_vr_put(mlxsw_sp, vr); 9447 } 9448 9449 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 9450 u16 *ul_rif_index) 9451 { 9452 struct mlxsw_sp_rif *ul_rif; 9453 int err = 0; 9454 9455 mutex_lock(&mlxsw_sp->router->lock); 9456 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9457 if (IS_ERR(ul_rif)) { 9458 err = PTR_ERR(ul_rif); 9459 goto out; 9460 } 9461 *ul_rif_index = ul_rif->rif_index; 9462 out: 9463 mutex_unlock(&mlxsw_sp->router->lock); 9464 return err; 9465 } 9466 9467 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 9468 { 9469 struct mlxsw_sp_rif *ul_rif; 9470 9471 mutex_lock(&mlxsw_sp->router->lock); 9472 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 9473 if (WARN_ON(!ul_rif)) 9474 goto out; 9475 9476 mlxsw_sp_ul_rif_put(ul_rif); 9477 out: 9478 mutex_unlock(&mlxsw_sp->router->lock); 9479 } 9480 9481 static int 9482 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif) 9483 { 9484 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9485 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9486 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9487 struct mlxsw_sp_rif *ul_rif; 9488 int err; 9489 9490 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9491 if (IS_ERR(ul_rif)) 9492 return PTR_ERR(ul_rif); 9493 9494 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 9495 if (err) 9496 goto err_loopback_op; 9497 9498 lb_rif->ul_vr_id = 0; 9499 lb_rif->ul_rif_id = ul_rif->rif_index; 9500 9501 return 0; 9502 9503 err_loopback_op: 9504 mlxsw_sp_ul_rif_put(ul_rif); 9505 return err; 9506 } 9507 9508 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9509 { 9510 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9511 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9512 struct mlxsw_sp_rif *ul_rif; 9513 9514 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 9515 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 9516 mlxsw_sp_ul_rif_put(ul_rif); 9517 } 9518 9519 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 9520 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9521 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9522 .setup = mlxsw_sp_rif_ipip_lb_setup, 9523 .configure = mlxsw_sp2_rif_ipip_lb_configure, 9524 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 9525 }; 9526 9527 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 9528 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9529 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9530 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9531 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 9532 }; 9533 9534 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 9535 { 9536 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9537 9538 mlxsw_sp->router->rifs = kcalloc(max_rifs, 9539 sizeof(struct mlxsw_sp_rif *), 9540 GFP_KERNEL); 9541 if (!mlxsw_sp->router->rifs) 9542 return -ENOMEM; 9543 9544 return 0; 9545 } 9546 9547 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 9548 { 9549 int i; 9550 9551 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 9552 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 9553 9554 kfree(mlxsw_sp->router->rifs); 9555 } 9556 9557 static int 9558 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 9559 { 9560 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 9561 9562 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 9563 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 9564 } 9565 9566 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 9567 { 9568 int err; 9569 9570 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr; 9571 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 9572 9573 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 9574 if (err) 9575 return err; 9576 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 9577 if (err) 9578 return err; 9579 9580 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9581 } 9582 9583 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9584 { 9585 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9586 } 9587 9588 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9589 { 9590 struct mlxsw_sp_router *router; 9591 9592 /* Flush pending FIB notifications and then flush the device's 9593 * table before requesting another dump. The FIB notification 9594 * block is unregistered, so no need to take RTNL. 9595 */ 9596 mlxsw_core_flush_owq(); 9597 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9598 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9599 } 9600 9601 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9602 struct mlxsw_sp_mp_hash_config { 9603 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 9604 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 9605 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 9606 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 9607 }; 9608 9609 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 9610 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 9611 9612 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 9613 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 9614 9615 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 9616 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 9617 9618 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 9619 { 9620 unsigned long *inner_headers = config->inner_headers; 9621 unsigned long *inner_fields = config->inner_fields; 9622 9623 /* IPv4 inner */ 9624 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9625 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9626 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9627 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9628 /* IPv6 inner */ 9629 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9630 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9631 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9632 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9633 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9634 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9635 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9636 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9637 } 9638 9639 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 9640 { 9641 unsigned long *headers = config->headers; 9642 unsigned long *fields = config->fields; 9643 9644 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 9645 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 9646 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 9647 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 9648 } 9649 9650 static void 9651 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 9652 u32 hash_fields) 9653 { 9654 unsigned long *inner_headers = config->inner_headers; 9655 unsigned long *inner_fields = config->inner_fields; 9656 9657 /* IPv4 Inner */ 9658 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9659 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9660 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 9661 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9662 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 9663 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9664 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9665 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 9666 /* IPv6 inner */ 9667 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9668 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9669 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 9670 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9671 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9672 } 9673 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 9674 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9675 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9676 } 9677 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9678 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9679 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 9680 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9681 /* L4 inner */ 9682 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 9683 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 9684 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 9685 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 9686 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 9687 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 9688 } 9689 9690 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 9691 struct mlxsw_sp_mp_hash_config *config) 9692 { 9693 struct net *net = mlxsw_sp_net(mlxsw_sp); 9694 unsigned long *headers = config->headers; 9695 unsigned long *fields = config->fields; 9696 u32 hash_fields; 9697 9698 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 9699 case 0: 9700 mlxsw_sp_mp4_hash_outer_addr(config); 9701 break; 9702 case 1: 9703 mlxsw_sp_mp4_hash_outer_addr(config); 9704 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 9705 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 9706 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9707 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9708 break; 9709 case 2: 9710 /* Outer */ 9711 mlxsw_sp_mp4_hash_outer_addr(config); 9712 /* Inner */ 9713 mlxsw_sp_mp_hash_inner_l3(config); 9714 break; 9715 case 3: 9716 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; 9717 /* Outer */ 9718 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 9719 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 9720 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 9721 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 9722 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 9723 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 9724 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 9725 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 9726 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 9727 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 9728 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9729 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 9730 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9731 /* Inner */ 9732 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 9733 break; 9734 } 9735 } 9736 9737 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 9738 { 9739 unsigned long *headers = config->headers; 9740 unsigned long *fields = config->fields; 9741 9742 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 9743 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 9744 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 9745 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 9746 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 9747 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 9748 } 9749 9750 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 9751 struct mlxsw_sp_mp_hash_config *config) 9752 { 9753 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 9754 unsigned long *headers = config->headers; 9755 unsigned long *fields = config->fields; 9756 9757 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 9758 case 0: 9759 mlxsw_sp_mp6_hash_outer_addr(config); 9760 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9761 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9762 break; 9763 case 1: 9764 mlxsw_sp_mp6_hash_outer_addr(config); 9765 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 9766 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9767 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9768 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9769 break; 9770 case 2: 9771 /* Outer */ 9772 mlxsw_sp_mp6_hash_outer_addr(config); 9773 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9774 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9775 /* Inner */ 9776 mlxsw_sp_mp_hash_inner_l3(config); 9777 break; 9778 case 3: 9779 /* Outer */ 9780 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 9781 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 9782 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 9783 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 9784 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 9785 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 9786 } 9787 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 9788 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 9789 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 9790 } 9791 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 9792 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 9793 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 9794 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 9795 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 9796 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 9797 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 9798 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 9799 /* Inner */ 9800 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 9801 break; 9802 } 9803 } 9804 9805 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9806 { 9807 struct mlxsw_sp_mp_hash_config config = {}; 9808 char recr2_pl[MLXSW_REG_RECR2_LEN]; 9809 unsigned long bit; 9810 u32 seed; 9811 9812 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 9813 mlxsw_reg_recr2_pack(recr2_pl, seed); 9814 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 9815 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 9816 9817 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 9818 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 9819 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 9820 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 9821 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 9822 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 9823 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 9824 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 9825 9826 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 9827 } 9828 #else 9829 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 9830 { 9831 return 0; 9832 } 9833 #endif 9834 9835 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 9836 { 9837 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 9838 unsigned int i; 9839 9840 MLXSW_REG_ZERO(rdpm, rdpm_pl); 9841 9842 /* HW is determining switch priority based on DSCP-bits, but the 9843 * kernel is still doing that based on the ToS. Since there's a 9844 * mismatch in bits we need to make sure to translate the right 9845 * value ToS would observe, skipping the 2 least-significant ECN bits. 9846 */ 9847 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 9848 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 9849 9850 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 9851 } 9852 9853 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 9854 { 9855 struct net *net = mlxsw_sp_net(mlxsw_sp); 9856 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 9857 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9858 u64 max_rifs; 9859 9860 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 9861 return -EIO; 9862 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9863 9864 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 9865 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 9866 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 9867 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9868 } 9869 9870 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 9871 { 9872 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 9873 9874 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 9875 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 9876 } 9877 9878 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 9879 .init = mlxsw_sp_router_ll_basic_init, 9880 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 9881 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 9882 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 9883 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 9884 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 9885 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 9886 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 9887 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 9888 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 9889 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 9890 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 9891 }; 9892 9893 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 9894 { 9895 size_t max_size = 0; 9896 int i; 9897 9898 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 9899 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 9900 9901 if (size > max_size) 9902 max_size = size; 9903 } 9904 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 9905 GFP_KERNEL); 9906 if (!router->ll_op_ctx) 9907 return -ENOMEM; 9908 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 9909 return 0; 9910 } 9911 9912 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 9913 { 9914 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 9915 kfree(router->ll_op_ctx); 9916 } 9917 9918 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 9919 { 9920 u16 lb_rif_index; 9921 int err; 9922 9923 /* Create a generic loopback RIF associated with the main table 9924 * (default VRF). Any table can be used, but the main table exists 9925 * anyway, so we do not waste resources. 9926 */ 9927 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 9928 &lb_rif_index); 9929 if (err) 9930 return err; 9931 9932 mlxsw_sp->router->lb_rif_index = lb_rif_index; 9933 9934 return 0; 9935 } 9936 9937 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 9938 { 9939 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 9940 } 9941 9942 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 9943 { 9944 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 9945 9946 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 9947 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 9948 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9949 9950 return 0; 9951 } 9952 9953 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 9954 .init = mlxsw_sp1_router_init, 9955 }; 9956 9957 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 9958 { 9959 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 9960 9961 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 9962 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 9963 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 9964 9965 return 0; 9966 } 9967 9968 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 9969 .init = mlxsw_sp2_router_init, 9970 }; 9971 9972 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 9973 struct netlink_ext_ack *extack) 9974 { 9975 struct mlxsw_sp_router *router; 9976 int err; 9977 9978 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 9979 if (!router) 9980 return -ENOMEM; 9981 mutex_init(&router->lock); 9982 mlxsw_sp->router = router; 9983 router->mlxsw_sp = mlxsw_sp; 9984 9985 err = mlxsw_sp->router_ops->init(mlxsw_sp); 9986 if (err) 9987 goto err_router_ops_init; 9988 9989 err = mlxsw_sp_router_xm_init(mlxsw_sp); 9990 if (err) 9991 goto err_xm_init; 9992 9993 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 9994 &mlxsw_sp_router_ll_xm_ops : 9995 &mlxsw_sp_router_ll_basic_ops; 9996 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 9997 9998 err = mlxsw_sp_router_ll_op_ctx_init(router); 9999 if (err) 10000 goto err_ll_op_ctx_init; 10001 10002 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 10003 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 10004 mlxsw_sp_nh_grp_activity_work); 10005 10006 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 10007 err = __mlxsw_sp_router_init(mlxsw_sp); 10008 if (err) 10009 goto err_router_init; 10010 10011 err = mlxsw_sp_rifs_init(mlxsw_sp); 10012 if (err) 10013 goto err_rifs_init; 10014 10015 err = mlxsw_sp_ipips_init(mlxsw_sp); 10016 if (err) 10017 goto err_ipips_init; 10018 10019 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 10020 &mlxsw_sp_nexthop_ht_params); 10021 if (err) 10022 goto err_nexthop_ht_init; 10023 10024 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 10025 &mlxsw_sp_nexthop_group_ht_params); 10026 if (err) 10027 goto err_nexthop_group_ht_init; 10028 10029 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 10030 err = mlxsw_sp_lpm_init(mlxsw_sp); 10031 if (err) 10032 goto err_lpm_init; 10033 10034 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 10035 if (err) 10036 goto err_mr_init; 10037 10038 err = mlxsw_sp_vrs_init(mlxsw_sp); 10039 if (err) 10040 goto err_vrs_init; 10041 10042 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 10043 if (err) 10044 goto err_lb_rif_init; 10045 10046 err = mlxsw_sp_neigh_init(mlxsw_sp); 10047 if (err) 10048 goto err_neigh_init; 10049 10050 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 10051 if (err) 10052 goto err_mp_hash_init; 10053 10054 err = mlxsw_sp_dscp_init(mlxsw_sp); 10055 if (err) 10056 goto err_dscp_init; 10057 10058 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 10059 INIT_LIST_HEAD(&router->fib_event_queue); 10060 spin_lock_init(&router->fib_event_queue_lock); 10061 10062 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 10063 err = register_inetaddr_notifier(&router->inetaddr_nb); 10064 if (err) 10065 goto err_register_inetaddr_notifier; 10066 10067 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 10068 err = register_inet6addr_notifier(&router->inet6addr_nb); 10069 if (err) 10070 goto err_register_inet6addr_notifier; 10071 10072 mlxsw_sp->router->netevent_nb.notifier_call = 10073 mlxsw_sp_router_netevent_event; 10074 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10075 if (err) 10076 goto err_register_netevent_notifier; 10077 10078 mlxsw_sp->router->nexthop_nb.notifier_call = 10079 mlxsw_sp_nexthop_obj_event; 10080 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10081 &mlxsw_sp->router->nexthop_nb, 10082 extack); 10083 if (err) 10084 goto err_register_nexthop_notifier; 10085 10086 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 10087 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10088 &mlxsw_sp->router->fib_nb, 10089 mlxsw_sp_router_fib_dump_flush, extack); 10090 if (err) 10091 goto err_register_fib_notifier; 10092 10093 return 0; 10094 10095 err_register_fib_notifier: 10096 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10097 &mlxsw_sp->router->nexthop_nb); 10098 err_register_nexthop_notifier: 10099 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10100 err_register_netevent_notifier: 10101 unregister_inet6addr_notifier(&router->inet6addr_nb); 10102 err_register_inet6addr_notifier: 10103 unregister_inetaddr_notifier(&router->inetaddr_nb); 10104 err_register_inetaddr_notifier: 10105 mlxsw_core_flush_owq(); 10106 WARN_ON(!list_empty(&router->fib_event_queue)); 10107 err_dscp_init: 10108 err_mp_hash_init: 10109 mlxsw_sp_neigh_fini(mlxsw_sp); 10110 err_neigh_init: 10111 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10112 err_lb_rif_init: 10113 mlxsw_sp_vrs_fini(mlxsw_sp); 10114 err_vrs_init: 10115 mlxsw_sp_mr_fini(mlxsw_sp); 10116 err_mr_init: 10117 mlxsw_sp_lpm_fini(mlxsw_sp); 10118 err_lpm_init: 10119 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10120 err_nexthop_group_ht_init: 10121 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10122 err_nexthop_ht_init: 10123 mlxsw_sp_ipips_fini(mlxsw_sp); 10124 err_ipips_init: 10125 mlxsw_sp_rifs_fini(mlxsw_sp); 10126 err_rifs_init: 10127 __mlxsw_sp_router_fini(mlxsw_sp); 10128 err_router_init: 10129 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10130 mlxsw_sp_router_ll_op_ctx_fini(router); 10131 err_ll_op_ctx_init: 10132 mlxsw_sp_router_xm_fini(mlxsw_sp); 10133 err_xm_init: 10134 err_router_ops_init: 10135 mutex_destroy(&mlxsw_sp->router->lock); 10136 kfree(mlxsw_sp->router); 10137 return err; 10138 } 10139 10140 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10141 { 10142 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10143 &mlxsw_sp->router->fib_nb); 10144 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10145 &mlxsw_sp->router->nexthop_nb); 10146 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10147 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 10148 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 10149 mlxsw_core_flush_owq(); 10150 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 10151 mlxsw_sp_neigh_fini(mlxsw_sp); 10152 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10153 mlxsw_sp_vrs_fini(mlxsw_sp); 10154 mlxsw_sp_mr_fini(mlxsw_sp); 10155 mlxsw_sp_lpm_fini(mlxsw_sp); 10156 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10157 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10158 mlxsw_sp_ipips_fini(mlxsw_sp); 10159 mlxsw_sp_rifs_fini(mlxsw_sp); 10160 __mlxsw_sp_router_fini(mlxsw_sp); 10161 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10162 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 10163 mlxsw_sp_router_xm_fini(mlxsw_sp); 10164 mutex_destroy(&mlxsw_sp->router->lock); 10165 kfree(mlxsw_sp->router); 10166 } 10167