1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/inet_dscp.h> 25 #include <net/ip_fib.h> 26 #include <net/ip6_fib.h> 27 #include <net/nexthop.h> 28 #include <net/fib_rules.h> 29 #include <net/ip_tunnels.h> 30 #include <net/l3mdev.h> 31 #include <net/addrconf.h> 32 #include <net/ndisc.h> 33 #include <net/ipv6.h> 34 #include <net/fib_notifier.h> 35 #include <net/switchdev.h> 36 37 #include "spectrum.h" 38 #include "core.h" 39 #include "reg.h" 40 #include "spectrum_cnt.h" 41 #include "spectrum_dpipe.h" 42 #include "spectrum_ipip.h" 43 #include "spectrum_mr.h" 44 #include "spectrum_mr_tcam.h" 45 #include "spectrum_router.h" 46 #include "spectrum_span.h" 47 48 struct mlxsw_sp_fib; 49 struct mlxsw_sp_vr; 50 struct mlxsw_sp_lpm_tree; 51 struct mlxsw_sp_rif_ops; 52 53 struct mlxsw_sp_rif { 54 struct list_head nexthop_list; 55 struct list_head neigh_list; 56 struct net_device *dev; /* NULL for underlay RIF */ 57 struct mlxsw_sp_fid *fid; 58 unsigned char addr[ETH_ALEN]; 59 int mtu; 60 u16 rif_index; 61 u8 mac_profile_id; 62 u16 vr_id; 63 const struct mlxsw_sp_rif_ops *ops; 64 struct mlxsw_sp *mlxsw_sp; 65 66 unsigned int counter_ingress; 67 bool counter_ingress_valid; 68 unsigned int counter_egress; 69 bool counter_egress_valid; 70 }; 71 72 struct mlxsw_sp_rif_params { 73 struct net_device *dev; 74 union { 75 u16 system_port; 76 u16 lag_id; 77 }; 78 u16 vid; 79 bool lag; 80 }; 81 82 struct mlxsw_sp_rif_subport { 83 struct mlxsw_sp_rif common; 84 refcount_t ref_count; 85 union { 86 u16 system_port; 87 u16 lag_id; 88 }; 89 u16 vid; 90 bool lag; 91 }; 92 93 struct mlxsw_sp_rif_ipip_lb { 94 struct mlxsw_sp_rif common; 95 struct mlxsw_sp_rif_ipip_lb_config lb_config; 96 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 97 u16 ul_rif_id; /* Reserved for Spectrum. */ 98 }; 99 100 struct mlxsw_sp_rif_params_ipip_lb { 101 struct mlxsw_sp_rif_params common; 102 struct mlxsw_sp_rif_ipip_lb_config lb_config; 103 }; 104 105 struct mlxsw_sp_rif_ops { 106 enum mlxsw_sp_rif_type type; 107 size_t rif_size; 108 109 void (*setup)(struct mlxsw_sp_rif *rif, 110 const struct mlxsw_sp_rif_params *params); 111 int (*configure)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*deconfigure)(struct mlxsw_sp_rif *rif); 114 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 115 struct netlink_ext_ack *extack); 116 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 117 }; 118 119 struct mlxsw_sp_rif_mac_profile { 120 unsigned char mac_prefix[ETH_ALEN]; 121 refcount_t ref_count; 122 u8 id; 123 }; 124 125 struct mlxsw_sp_router_ops { 126 int (*init)(struct mlxsw_sp *mlxsw_sp); 127 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 128 }; 129 130 static struct mlxsw_sp_rif * 131 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 132 const struct net_device *dev); 133 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 134 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 135 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 136 struct mlxsw_sp_lpm_tree *lpm_tree); 137 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 138 const struct mlxsw_sp_fib *fib, 139 u8 tree_id); 140 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 141 const struct mlxsw_sp_fib *fib); 142 143 static unsigned int * 144 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 145 enum mlxsw_sp_rif_counter_dir dir) 146 { 147 switch (dir) { 148 case MLXSW_SP_RIF_COUNTER_EGRESS: 149 return &rif->counter_egress; 150 case MLXSW_SP_RIF_COUNTER_INGRESS: 151 return &rif->counter_ingress; 152 } 153 return NULL; 154 } 155 156 static bool 157 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 158 enum mlxsw_sp_rif_counter_dir dir) 159 { 160 switch (dir) { 161 case MLXSW_SP_RIF_COUNTER_EGRESS: 162 return rif->counter_egress_valid; 163 case MLXSW_SP_RIF_COUNTER_INGRESS: 164 return rif->counter_ingress_valid; 165 } 166 return false; 167 } 168 169 static void 170 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 171 enum mlxsw_sp_rif_counter_dir dir, 172 bool valid) 173 { 174 switch (dir) { 175 case MLXSW_SP_RIF_COUNTER_EGRESS: 176 rif->counter_egress_valid = valid; 177 break; 178 case MLXSW_SP_RIF_COUNTER_INGRESS: 179 rif->counter_ingress_valid = valid; 180 break; 181 } 182 } 183 184 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 185 unsigned int counter_index, bool enable, 186 enum mlxsw_sp_rif_counter_dir dir) 187 { 188 char ritr_pl[MLXSW_REG_RITR_LEN]; 189 bool is_egress = false; 190 int err; 191 192 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 193 is_egress = true; 194 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 195 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 196 if (err) 197 return err; 198 199 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 200 is_egress); 201 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 202 } 203 204 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 205 struct mlxsw_sp_rif *rif, 206 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 207 { 208 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 209 unsigned int *p_counter_index; 210 bool valid; 211 int err; 212 213 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 214 if (!valid) 215 return -EINVAL; 216 217 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 218 if (!p_counter_index) 219 return -EINVAL; 220 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 221 MLXSW_REG_RICNT_OPCODE_NOP); 222 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 223 if (err) 224 return err; 225 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 226 return 0; 227 } 228 229 struct mlxsw_sp_rif_counter_set_basic { 230 u64 good_unicast_packets; 231 u64 good_multicast_packets; 232 u64 good_broadcast_packets; 233 u64 good_unicast_bytes; 234 u64 good_multicast_bytes; 235 u64 good_broadcast_bytes; 236 u64 error_packets; 237 u64 discard_packets; 238 u64 error_bytes; 239 u64 discard_bytes; 240 }; 241 242 static int 243 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 244 enum mlxsw_sp_rif_counter_dir dir, 245 struct mlxsw_sp_rif_counter_set_basic *set) 246 { 247 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 248 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 249 unsigned int *p_counter_index; 250 int err; 251 252 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 253 return -EINVAL; 254 255 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 256 if (!p_counter_index) 257 return -EINVAL; 258 259 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 260 MLXSW_REG_RICNT_OPCODE_CLEAR); 261 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 262 if (err) 263 return err; 264 265 if (!set) 266 return 0; 267 268 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 269 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 270 271 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 272 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 273 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 274 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 275 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 276 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 277 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 278 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 279 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 280 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 281 282 #undef MLXSW_SP_RIF_COUNTER_EXTRACT 283 284 return 0; 285 } 286 287 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 288 unsigned int counter_index) 289 { 290 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 291 292 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 293 MLXSW_REG_RICNT_OPCODE_CLEAR); 294 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 295 } 296 297 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 298 enum mlxsw_sp_rif_counter_dir dir) 299 { 300 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 301 unsigned int *p_counter_index; 302 int err; 303 304 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 305 return 0; 306 307 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 308 if (!p_counter_index) 309 return -EINVAL; 310 311 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 312 p_counter_index); 313 if (err) 314 return err; 315 316 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 317 if (err) 318 goto err_counter_clear; 319 320 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 321 *p_counter_index, true, dir); 322 if (err) 323 goto err_counter_edit; 324 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 325 return 0; 326 327 err_counter_edit: 328 err_counter_clear: 329 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 330 *p_counter_index); 331 return err; 332 } 333 334 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 335 enum mlxsw_sp_rif_counter_dir dir) 336 { 337 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 338 unsigned int *p_counter_index; 339 340 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 341 return; 342 343 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 344 if (WARN_ON(!p_counter_index)) 345 return; 346 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 347 *p_counter_index, false, dir); 348 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 349 *p_counter_index); 350 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 351 } 352 353 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 354 { 355 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 356 struct devlink *devlink; 357 358 devlink = priv_to_devlink(mlxsw_sp->core); 359 if (!devlink_dpipe_table_counter_enabled(devlink, 360 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 361 return; 362 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 363 } 364 365 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 366 { 367 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 368 } 369 370 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 371 372 struct mlxsw_sp_prefix_usage { 373 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 374 }; 375 376 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 377 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 378 379 static bool 380 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 381 struct mlxsw_sp_prefix_usage *prefix_usage2) 382 { 383 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 384 } 385 386 static void 387 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 388 struct mlxsw_sp_prefix_usage *prefix_usage2) 389 { 390 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 391 } 392 393 static void 394 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 395 unsigned char prefix_len) 396 { 397 set_bit(prefix_len, prefix_usage->b); 398 } 399 400 static void 401 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 402 unsigned char prefix_len) 403 { 404 clear_bit(prefix_len, prefix_usage->b); 405 } 406 407 struct mlxsw_sp_fib_key { 408 unsigned char addr[sizeof(struct in6_addr)]; 409 unsigned char prefix_len; 410 }; 411 412 enum mlxsw_sp_fib_entry_type { 413 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 414 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 415 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 416 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 417 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 418 419 /* This is a special case of local delivery, where a packet should be 420 * decapsulated on reception. Note that there is no corresponding ENCAP, 421 * because that's a type of next hop, not of FIB entry. (There can be 422 * several next hops in a REMOTE entry, and some of them may be 423 * encapsulating entries.) 424 */ 425 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 426 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 427 }; 428 429 struct mlxsw_sp_nexthop_group_info; 430 struct mlxsw_sp_nexthop_group; 431 struct mlxsw_sp_fib_entry; 432 433 struct mlxsw_sp_fib_node { 434 struct mlxsw_sp_fib_entry *fib_entry; 435 struct list_head list; 436 struct rhash_head ht_node; 437 struct mlxsw_sp_fib *fib; 438 struct mlxsw_sp_fib_key key; 439 }; 440 441 struct mlxsw_sp_fib_entry_decap { 442 struct mlxsw_sp_ipip_entry *ipip_entry; 443 u32 tunnel_index; 444 }; 445 446 static struct mlxsw_sp_fib_entry_priv * 447 mlxsw_sp_fib_entry_priv_create(const struct mlxsw_sp_router_ll_ops *ll_ops) 448 { 449 struct mlxsw_sp_fib_entry_priv *priv; 450 451 if (!ll_ops->fib_entry_priv_size) 452 /* No need to have priv */ 453 return NULL; 454 455 priv = kzalloc(sizeof(*priv) + ll_ops->fib_entry_priv_size, GFP_KERNEL); 456 if (!priv) 457 return ERR_PTR(-ENOMEM); 458 refcount_set(&priv->refcnt, 1); 459 return priv; 460 } 461 462 static void 463 mlxsw_sp_fib_entry_priv_destroy(struct mlxsw_sp_fib_entry_priv *priv) 464 { 465 kfree(priv); 466 } 467 468 static void mlxsw_sp_fib_entry_priv_hold(struct mlxsw_sp_fib_entry_priv *priv) 469 { 470 refcount_inc(&priv->refcnt); 471 } 472 473 static void mlxsw_sp_fib_entry_priv_put(struct mlxsw_sp_fib_entry_priv *priv) 474 { 475 if (!priv || !refcount_dec_and_test(&priv->refcnt)) 476 return; 477 mlxsw_sp_fib_entry_priv_destroy(priv); 478 } 479 480 static void mlxsw_sp_fib_entry_op_ctx_priv_hold(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 481 struct mlxsw_sp_fib_entry_priv *priv) 482 { 483 if (!priv) 484 return; 485 mlxsw_sp_fib_entry_priv_hold(priv); 486 list_add(&priv->list, &op_ctx->fib_entry_priv_list); 487 } 488 489 static void mlxsw_sp_fib_entry_op_ctx_priv_put_all(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 490 { 491 struct mlxsw_sp_fib_entry_priv *priv, *tmp; 492 493 list_for_each_entry_safe(priv, tmp, &op_ctx->fib_entry_priv_list, list) 494 mlxsw_sp_fib_entry_priv_put(priv); 495 INIT_LIST_HEAD(&op_ctx->fib_entry_priv_list); 496 } 497 498 struct mlxsw_sp_fib_entry { 499 struct mlxsw_sp_fib_node *fib_node; 500 enum mlxsw_sp_fib_entry_type type; 501 struct list_head nexthop_group_node; 502 struct mlxsw_sp_nexthop_group *nh_group; 503 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 504 struct mlxsw_sp_fib_entry_priv *priv; 505 }; 506 507 struct mlxsw_sp_fib4_entry { 508 struct mlxsw_sp_fib_entry common; 509 struct fib_info *fi; 510 u32 tb_id; 511 dscp_t dscp; 512 u8 type; 513 }; 514 515 struct mlxsw_sp_fib6_entry { 516 struct mlxsw_sp_fib_entry common; 517 struct list_head rt6_list; 518 unsigned int nrt6; 519 }; 520 521 struct mlxsw_sp_rt6 { 522 struct list_head list; 523 struct fib6_info *rt; 524 }; 525 526 struct mlxsw_sp_lpm_tree { 527 u8 id; /* tree ID */ 528 unsigned int ref_count; 529 enum mlxsw_sp_l3proto proto; 530 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 531 struct mlxsw_sp_prefix_usage prefix_usage; 532 }; 533 534 struct mlxsw_sp_fib { 535 struct rhashtable ht; 536 struct list_head node_list; 537 struct mlxsw_sp_vr *vr; 538 struct mlxsw_sp_lpm_tree *lpm_tree; 539 enum mlxsw_sp_l3proto proto; 540 const struct mlxsw_sp_router_ll_ops *ll_ops; 541 }; 542 543 struct mlxsw_sp_vr { 544 u16 id; /* virtual router ID */ 545 u32 tb_id; /* kernel fib table id */ 546 unsigned int rif_count; 547 struct mlxsw_sp_fib *fib4; 548 struct mlxsw_sp_fib *fib6; 549 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 550 struct mlxsw_sp_rif *ul_rif; 551 refcount_t ul_rif_refcnt; 552 }; 553 554 static int mlxsw_sp_router_ll_basic_init(struct mlxsw_sp *mlxsw_sp, u16 vr_id, 555 enum mlxsw_sp_l3proto proto) 556 { 557 return 0; 558 } 559 560 static int mlxsw_sp_router_ll_basic_ralta_write(struct mlxsw_sp *mlxsw_sp, char *xralta_pl) 561 { 562 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), 563 xralta_pl + MLXSW_REG_XRALTA_RALTA_OFFSET); 564 } 565 566 static int mlxsw_sp_router_ll_basic_ralst_write(struct mlxsw_sp *mlxsw_sp, char *xralst_pl) 567 { 568 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), 569 xralst_pl + MLXSW_REG_XRALST_RALST_OFFSET); 570 } 571 572 static int mlxsw_sp_router_ll_basic_raltb_write(struct mlxsw_sp *mlxsw_sp, char *xraltb_pl) 573 { 574 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), 575 xraltb_pl + MLXSW_REG_XRALTB_RALTB_OFFSET); 576 } 577 578 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 579 580 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 581 struct mlxsw_sp_vr *vr, 582 enum mlxsw_sp_l3proto proto) 583 { 584 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 585 struct mlxsw_sp_lpm_tree *lpm_tree; 586 struct mlxsw_sp_fib *fib; 587 int err; 588 589 err = ll_ops->init(mlxsw_sp, vr->id, proto); 590 if (err) 591 return ERR_PTR(err); 592 593 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 594 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 595 if (!fib) 596 return ERR_PTR(-ENOMEM); 597 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 598 if (err) 599 goto err_rhashtable_init; 600 INIT_LIST_HEAD(&fib->node_list); 601 fib->proto = proto; 602 fib->vr = vr; 603 fib->lpm_tree = lpm_tree; 604 fib->ll_ops = ll_ops; 605 mlxsw_sp_lpm_tree_hold(lpm_tree); 606 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 607 if (err) 608 goto err_lpm_tree_bind; 609 return fib; 610 611 err_lpm_tree_bind: 612 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 613 err_rhashtable_init: 614 kfree(fib); 615 return ERR_PTR(err); 616 } 617 618 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 619 struct mlxsw_sp_fib *fib) 620 { 621 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 622 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 623 WARN_ON(!list_empty(&fib->node_list)); 624 rhashtable_destroy(&fib->ht); 625 kfree(fib); 626 } 627 628 static struct mlxsw_sp_lpm_tree * 629 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 630 { 631 static struct mlxsw_sp_lpm_tree *lpm_tree; 632 int i; 633 634 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 635 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 636 if (lpm_tree->ref_count == 0) 637 return lpm_tree; 638 } 639 return NULL; 640 } 641 642 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 643 const struct mlxsw_sp_router_ll_ops *ll_ops, 644 struct mlxsw_sp_lpm_tree *lpm_tree) 645 { 646 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 647 648 mlxsw_reg_xralta_pack(xralta_pl, true, 649 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 650 lpm_tree->id); 651 return ll_ops->ralta_write(mlxsw_sp, xralta_pl); 652 } 653 654 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 655 const struct mlxsw_sp_router_ll_ops *ll_ops, 656 struct mlxsw_sp_lpm_tree *lpm_tree) 657 { 658 char xralta_pl[MLXSW_REG_XRALTA_LEN]; 659 660 mlxsw_reg_xralta_pack(xralta_pl, false, 661 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 662 lpm_tree->id); 663 ll_ops->ralta_write(mlxsw_sp, xralta_pl); 664 } 665 666 static int 667 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 668 const struct mlxsw_sp_router_ll_ops *ll_ops, 669 struct mlxsw_sp_prefix_usage *prefix_usage, 670 struct mlxsw_sp_lpm_tree *lpm_tree) 671 { 672 char xralst_pl[MLXSW_REG_XRALST_LEN]; 673 u8 root_bin = 0; 674 u8 prefix; 675 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 676 677 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 678 root_bin = prefix; 679 680 mlxsw_reg_xralst_pack(xralst_pl, root_bin, lpm_tree->id); 681 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 682 if (prefix == 0) 683 continue; 684 mlxsw_reg_xralst_bin_pack(xralst_pl, prefix, last_prefix, 685 MLXSW_REG_RALST_BIN_NO_CHILD); 686 last_prefix = prefix; 687 } 688 return ll_ops->ralst_write(mlxsw_sp, xralst_pl); 689 } 690 691 static struct mlxsw_sp_lpm_tree * 692 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 693 const struct mlxsw_sp_router_ll_ops *ll_ops, 694 struct mlxsw_sp_prefix_usage *prefix_usage, 695 enum mlxsw_sp_l3proto proto) 696 { 697 struct mlxsw_sp_lpm_tree *lpm_tree; 698 int err; 699 700 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 701 if (!lpm_tree) 702 return ERR_PTR(-EBUSY); 703 lpm_tree->proto = proto; 704 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, ll_ops, lpm_tree); 705 if (err) 706 return ERR_PTR(err); 707 708 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, ll_ops, prefix_usage, lpm_tree); 709 if (err) 710 goto err_left_struct_set; 711 memcpy(&lpm_tree->prefix_usage, prefix_usage, 712 sizeof(lpm_tree->prefix_usage)); 713 memset(&lpm_tree->prefix_ref_count, 0, 714 sizeof(lpm_tree->prefix_ref_count)); 715 lpm_tree->ref_count = 1; 716 return lpm_tree; 717 718 err_left_struct_set: 719 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 720 return ERR_PTR(err); 721 } 722 723 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 724 const struct mlxsw_sp_router_ll_ops *ll_ops, 725 struct mlxsw_sp_lpm_tree *lpm_tree) 726 { 727 mlxsw_sp_lpm_tree_free(mlxsw_sp, ll_ops, lpm_tree); 728 } 729 730 static struct mlxsw_sp_lpm_tree * 731 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 732 struct mlxsw_sp_prefix_usage *prefix_usage, 733 enum mlxsw_sp_l3proto proto) 734 { 735 const struct mlxsw_sp_router_ll_ops *ll_ops = mlxsw_sp->router->proto_ll_ops[proto]; 736 struct mlxsw_sp_lpm_tree *lpm_tree; 737 int i; 738 739 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 740 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 741 if (lpm_tree->ref_count != 0 && 742 lpm_tree->proto == proto && 743 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 744 prefix_usage)) { 745 mlxsw_sp_lpm_tree_hold(lpm_tree); 746 return lpm_tree; 747 } 748 } 749 return mlxsw_sp_lpm_tree_create(mlxsw_sp, ll_ops, prefix_usage, proto); 750 } 751 752 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 753 { 754 lpm_tree->ref_count++; 755 } 756 757 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 758 struct mlxsw_sp_lpm_tree *lpm_tree) 759 { 760 const struct mlxsw_sp_router_ll_ops *ll_ops = 761 mlxsw_sp->router->proto_ll_ops[lpm_tree->proto]; 762 763 if (--lpm_tree->ref_count == 0) 764 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, ll_ops, lpm_tree); 765 } 766 767 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 768 769 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 770 { 771 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 772 struct mlxsw_sp_lpm_tree *lpm_tree; 773 u64 max_trees; 774 int err, i; 775 776 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 777 return -EIO; 778 779 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 780 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 781 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 782 sizeof(struct mlxsw_sp_lpm_tree), 783 GFP_KERNEL); 784 if (!mlxsw_sp->router->lpm.trees) 785 return -ENOMEM; 786 787 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 788 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 789 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 790 } 791 792 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 793 MLXSW_SP_L3_PROTO_IPV4); 794 if (IS_ERR(lpm_tree)) { 795 err = PTR_ERR(lpm_tree); 796 goto err_ipv4_tree_get; 797 } 798 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 799 800 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 801 MLXSW_SP_L3_PROTO_IPV6); 802 if (IS_ERR(lpm_tree)) { 803 err = PTR_ERR(lpm_tree); 804 goto err_ipv6_tree_get; 805 } 806 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 807 808 return 0; 809 810 err_ipv6_tree_get: 811 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 812 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 813 err_ipv4_tree_get: 814 kfree(mlxsw_sp->router->lpm.trees); 815 return err; 816 } 817 818 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 819 { 820 struct mlxsw_sp_lpm_tree *lpm_tree; 821 822 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 823 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 824 825 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 826 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 827 828 kfree(mlxsw_sp->router->lpm.trees); 829 } 830 831 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 832 { 833 return !!vr->fib4 || !!vr->fib6 || 834 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 835 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 836 } 837 838 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 839 { 840 struct mlxsw_sp_vr *vr; 841 int i; 842 843 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 844 vr = &mlxsw_sp->router->vrs[i]; 845 if (!mlxsw_sp_vr_is_used(vr)) 846 return vr; 847 } 848 return NULL; 849 } 850 851 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 852 const struct mlxsw_sp_fib *fib, u8 tree_id) 853 { 854 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 855 856 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 857 (enum mlxsw_reg_ralxx_protocol) fib->proto, 858 tree_id); 859 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 860 } 861 862 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 863 const struct mlxsw_sp_fib *fib) 864 { 865 char xraltb_pl[MLXSW_REG_XRALTB_LEN]; 866 867 /* Bind to tree 0 which is default */ 868 mlxsw_reg_xraltb_pack(xraltb_pl, fib->vr->id, 869 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 870 return fib->ll_ops->raltb_write(mlxsw_sp, xraltb_pl); 871 } 872 873 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 874 { 875 /* For our purpose, squash main, default and local tables into one */ 876 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 877 tb_id = RT_TABLE_MAIN; 878 return tb_id; 879 } 880 881 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 882 u32 tb_id) 883 { 884 struct mlxsw_sp_vr *vr; 885 int i; 886 887 tb_id = mlxsw_sp_fix_tb_id(tb_id); 888 889 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 890 vr = &mlxsw_sp->router->vrs[i]; 891 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 892 return vr; 893 } 894 return NULL; 895 } 896 897 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 898 u16 *vr_id) 899 { 900 struct mlxsw_sp_vr *vr; 901 int err = 0; 902 903 mutex_lock(&mlxsw_sp->router->lock); 904 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 905 if (!vr) { 906 err = -ESRCH; 907 goto out; 908 } 909 *vr_id = vr->id; 910 out: 911 mutex_unlock(&mlxsw_sp->router->lock); 912 return err; 913 } 914 915 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 916 enum mlxsw_sp_l3proto proto) 917 { 918 switch (proto) { 919 case MLXSW_SP_L3_PROTO_IPV4: 920 return vr->fib4; 921 case MLXSW_SP_L3_PROTO_IPV6: 922 return vr->fib6; 923 } 924 return NULL; 925 } 926 927 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 928 u32 tb_id, 929 struct netlink_ext_ack *extack) 930 { 931 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 932 struct mlxsw_sp_fib *fib4; 933 struct mlxsw_sp_fib *fib6; 934 struct mlxsw_sp_vr *vr; 935 int err; 936 937 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 938 if (!vr) { 939 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 940 return ERR_PTR(-EBUSY); 941 } 942 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 943 if (IS_ERR(fib4)) 944 return ERR_CAST(fib4); 945 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 946 if (IS_ERR(fib6)) { 947 err = PTR_ERR(fib6); 948 goto err_fib6_create; 949 } 950 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 951 MLXSW_SP_L3_PROTO_IPV4); 952 if (IS_ERR(mr4_table)) { 953 err = PTR_ERR(mr4_table); 954 goto err_mr4_table_create; 955 } 956 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 957 MLXSW_SP_L3_PROTO_IPV6); 958 if (IS_ERR(mr6_table)) { 959 err = PTR_ERR(mr6_table); 960 goto err_mr6_table_create; 961 } 962 963 vr->fib4 = fib4; 964 vr->fib6 = fib6; 965 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 966 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 967 vr->tb_id = tb_id; 968 return vr; 969 970 err_mr6_table_create: 971 mlxsw_sp_mr_table_destroy(mr4_table); 972 err_mr4_table_create: 973 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 974 err_fib6_create: 975 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 976 return ERR_PTR(err); 977 } 978 979 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 980 struct mlxsw_sp_vr *vr) 981 { 982 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 983 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 984 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 985 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 986 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 987 vr->fib6 = NULL; 988 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 989 vr->fib4 = NULL; 990 } 991 992 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 993 struct netlink_ext_ack *extack) 994 { 995 struct mlxsw_sp_vr *vr; 996 997 tb_id = mlxsw_sp_fix_tb_id(tb_id); 998 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 999 if (!vr) 1000 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 1001 return vr; 1002 } 1003 1004 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 1005 { 1006 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 1007 list_empty(&vr->fib6->node_list) && 1008 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 1009 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 1010 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 1011 } 1012 1013 static bool 1014 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 1015 enum mlxsw_sp_l3proto proto, u8 tree_id) 1016 { 1017 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 1018 1019 if (!mlxsw_sp_vr_is_used(vr)) 1020 return false; 1021 if (fib->lpm_tree->id == tree_id) 1022 return true; 1023 return false; 1024 } 1025 1026 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 1027 struct mlxsw_sp_fib *fib, 1028 struct mlxsw_sp_lpm_tree *new_tree) 1029 { 1030 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 1031 int err; 1032 1033 fib->lpm_tree = new_tree; 1034 mlxsw_sp_lpm_tree_hold(new_tree); 1035 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 1036 if (err) 1037 goto err_tree_bind; 1038 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1039 return 0; 1040 1041 err_tree_bind: 1042 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 1043 fib->lpm_tree = old_tree; 1044 return err; 1045 } 1046 1047 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 1048 struct mlxsw_sp_fib *fib, 1049 struct mlxsw_sp_lpm_tree *new_tree) 1050 { 1051 enum mlxsw_sp_l3proto proto = fib->proto; 1052 struct mlxsw_sp_lpm_tree *old_tree; 1053 u8 old_id, new_id = new_tree->id; 1054 struct mlxsw_sp_vr *vr; 1055 int i, err; 1056 1057 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 1058 old_id = old_tree->id; 1059 1060 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 1061 vr = &mlxsw_sp->router->vrs[i]; 1062 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1063 continue; 1064 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1065 mlxsw_sp_vr_fib(vr, proto), 1066 new_tree); 1067 if (err) 1068 goto err_tree_replace; 1069 } 1070 1071 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1072 sizeof(new_tree->prefix_ref_count)); 1073 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1074 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1075 1076 return 0; 1077 1078 err_tree_replace: 1079 for (i--; i >= 0; i--) { 1080 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1081 continue; 1082 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1083 mlxsw_sp_vr_fib(vr, proto), 1084 old_tree); 1085 } 1086 return err; 1087 } 1088 1089 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1090 { 1091 struct mlxsw_sp_vr *vr; 1092 u64 max_vrs; 1093 int i; 1094 1095 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1096 return -EIO; 1097 1098 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1099 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1100 GFP_KERNEL); 1101 if (!mlxsw_sp->router->vrs) 1102 return -ENOMEM; 1103 1104 for (i = 0; i < max_vrs; i++) { 1105 vr = &mlxsw_sp->router->vrs[i]; 1106 vr->id = i; 1107 } 1108 1109 return 0; 1110 } 1111 1112 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1113 1114 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1115 { 1116 /* At this stage we're guaranteed not to have new incoming 1117 * FIB notifications and the work queue is free from FIBs 1118 * sitting on top of mlxsw netdevs. However, we can still 1119 * have other FIBs queued. Flush the queue before flushing 1120 * the device's tables. No need for locks, as we're the only 1121 * writer. 1122 */ 1123 mlxsw_core_flush_owq(); 1124 mlxsw_sp_router_fib_flush(mlxsw_sp); 1125 kfree(mlxsw_sp->router->vrs); 1126 } 1127 1128 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1129 { 1130 struct net_device *d; 1131 u32 tb_id; 1132 1133 rcu_read_lock(); 1134 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1135 if (d) 1136 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1137 else 1138 tb_id = RT_TABLE_MAIN; 1139 rcu_read_unlock(); 1140 1141 return tb_id; 1142 } 1143 1144 static struct mlxsw_sp_rif * 1145 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1146 const struct mlxsw_sp_rif_params *params, 1147 struct netlink_ext_ack *extack); 1148 1149 static struct mlxsw_sp_rif_ipip_lb * 1150 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1151 enum mlxsw_sp_ipip_type ipipt, 1152 struct net_device *ol_dev, 1153 struct netlink_ext_ack *extack) 1154 { 1155 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1156 const struct mlxsw_sp_ipip_ops *ipip_ops; 1157 struct mlxsw_sp_rif *rif; 1158 1159 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1160 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1161 .common.dev = ol_dev, 1162 .common.lag = false, 1163 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1164 }; 1165 1166 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1167 if (IS_ERR(rif)) 1168 return ERR_CAST(rif); 1169 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1170 } 1171 1172 static struct mlxsw_sp_ipip_entry * 1173 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1174 enum mlxsw_sp_ipip_type ipipt, 1175 struct net_device *ol_dev) 1176 { 1177 const struct mlxsw_sp_ipip_ops *ipip_ops; 1178 struct mlxsw_sp_ipip_entry *ipip_entry; 1179 struct mlxsw_sp_ipip_entry *ret = NULL; 1180 int err; 1181 1182 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1183 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1184 if (!ipip_entry) 1185 return ERR_PTR(-ENOMEM); 1186 1187 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1188 ol_dev, NULL); 1189 if (IS_ERR(ipip_entry->ol_lb)) { 1190 ret = ERR_CAST(ipip_entry->ol_lb); 1191 goto err_ol_ipip_lb_create; 1192 } 1193 1194 ipip_entry->ipipt = ipipt; 1195 ipip_entry->ol_dev = ol_dev; 1196 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1197 1198 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1199 if (err) { 1200 ret = ERR_PTR(err); 1201 goto err_rem_ip_addr_set; 1202 } 1203 1204 return ipip_entry; 1205 1206 err_rem_ip_addr_set: 1207 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1208 err_ol_ipip_lb_create: 1209 kfree(ipip_entry); 1210 return ret; 1211 } 1212 1213 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1214 struct mlxsw_sp_ipip_entry *ipip_entry) 1215 { 1216 const struct mlxsw_sp_ipip_ops *ipip_ops = 1217 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1218 1219 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1220 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1221 kfree(ipip_entry); 1222 } 1223 1224 static bool 1225 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1226 const enum mlxsw_sp_l3proto ul_proto, 1227 union mlxsw_sp_l3addr saddr, 1228 u32 ul_tb_id, 1229 struct mlxsw_sp_ipip_entry *ipip_entry) 1230 { 1231 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1232 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1233 union mlxsw_sp_l3addr tun_saddr; 1234 1235 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1236 return false; 1237 1238 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1239 return tun_ul_tb_id == ul_tb_id && 1240 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1241 } 1242 1243 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1244 enum mlxsw_sp_ipip_type ipipt) 1245 { 1246 const struct mlxsw_sp_ipip_ops *ipip_ops; 1247 1248 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1249 1250 /* Not all tunnels require to increase the default pasing depth 1251 * (96 bytes). 1252 */ 1253 if (ipip_ops->inc_parsing_depth) 1254 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1255 1256 return 0; 1257 } 1258 1259 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1260 enum mlxsw_sp_ipip_type ipipt) 1261 { 1262 const struct mlxsw_sp_ipip_ops *ipip_ops = 1263 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1264 1265 if (ipip_ops->inc_parsing_depth) 1266 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1267 } 1268 1269 static int 1270 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1271 struct mlxsw_sp_fib_entry *fib_entry, 1272 struct mlxsw_sp_ipip_entry *ipip_entry) 1273 { 1274 u32 tunnel_index; 1275 int err; 1276 1277 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1278 1, &tunnel_index); 1279 if (err) 1280 return err; 1281 1282 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1283 ipip_entry->ipipt); 1284 if (err) 1285 goto err_parsing_depth_inc; 1286 1287 ipip_entry->decap_fib_entry = fib_entry; 1288 fib_entry->decap.ipip_entry = ipip_entry; 1289 fib_entry->decap.tunnel_index = tunnel_index; 1290 1291 return 0; 1292 1293 err_parsing_depth_inc: 1294 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1295 fib_entry->decap.tunnel_index); 1296 return err; 1297 } 1298 1299 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1300 struct mlxsw_sp_fib_entry *fib_entry) 1301 { 1302 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1303 1304 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1305 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1306 fib_entry->decap.ipip_entry = NULL; 1307 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1308 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1309 1, fib_entry->decap.tunnel_index); 1310 } 1311 1312 static struct mlxsw_sp_fib_node * 1313 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1314 size_t addr_len, unsigned char prefix_len); 1315 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1316 struct mlxsw_sp_fib_entry *fib_entry); 1317 1318 static void 1319 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1320 struct mlxsw_sp_ipip_entry *ipip_entry) 1321 { 1322 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1323 1324 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1325 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1326 1327 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1328 } 1329 1330 static void 1331 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1332 struct mlxsw_sp_ipip_entry *ipip_entry, 1333 struct mlxsw_sp_fib_entry *decap_fib_entry) 1334 { 1335 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1336 ipip_entry)) 1337 return; 1338 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1339 1340 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1341 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1342 } 1343 1344 static struct mlxsw_sp_fib_entry * 1345 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1346 enum mlxsw_sp_l3proto proto, 1347 const union mlxsw_sp_l3addr *addr, 1348 enum mlxsw_sp_fib_entry_type type) 1349 { 1350 struct mlxsw_sp_fib_node *fib_node; 1351 unsigned char addr_prefix_len; 1352 struct mlxsw_sp_fib *fib; 1353 struct mlxsw_sp_vr *vr; 1354 const void *addrp; 1355 size_t addr_len; 1356 u32 addr4; 1357 1358 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1359 if (!vr) 1360 return NULL; 1361 fib = mlxsw_sp_vr_fib(vr, proto); 1362 1363 switch (proto) { 1364 case MLXSW_SP_L3_PROTO_IPV4: 1365 addr4 = be32_to_cpu(addr->addr4); 1366 addrp = &addr4; 1367 addr_len = 4; 1368 addr_prefix_len = 32; 1369 break; 1370 case MLXSW_SP_L3_PROTO_IPV6: 1371 addrp = &addr->addr6; 1372 addr_len = 16; 1373 addr_prefix_len = 128; 1374 break; 1375 default: 1376 WARN_ON(1); 1377 return NULL; 1378 } 1379 1380 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1381 addr_prefix_len); 1382 if (!fib_node || fib_node->fib_entry->type != type) 1383 return NULL; 1384 1385 return fib_node->fib_entry; 1386 } 1387 1388 /* Given an IPIP entry, find the corresponding decap route. */ 1389 static struct mlxsw_sp_fib_entry * 1390 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1391 struct mlxsw_sp_ipip_entry *ipip_entry) 1392 { 1393 static struct mlxsw_sp_fib_node *fib_node; 1394 const struct mlxsw_sp_ipip_ops *ipip_ops; 1395 unsigned char saddr_prefix_len; 1396 union mlxsw_sp_l3addr saddr; 1397 struct mlxsw_sp_fib *ul_fib; 1398 struct mlxsw_sp_vr *ul_vr; 1399 const void *saddrp; 1400 size_t saddr_len; 1401 u32 ul_tb_id; 1402 u32 saddr4; 1403 1404 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1405 1406 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1407 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1408 if (!ul_vr) 1409 return NULL; 1410 1411 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1412 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1413 ipip_entry->ol_dev); 1414 1415 switch (ipip_ops->ul_proto) { 1416 case MLXSW_SP_L3_PROTO_IPV4: 1417 saddr4 = be32_to_cpu(saddr.addr4); 1418 saddrp = &saddr4; 1419 saddr_len = 4; 1420 saddr_prefix_len = 32; 1421 break; 1422 case MLXSW_SP_L3_PROTO_IPV6: 1423 saddrp = &saddr.addr6; 1424 saddr_len = 16; 1425 saddr_prefix_len = 128; 1426 break; 1427 default: 1428 WARN_ON(1); 1429 return NULL; 1430 } 1431 1432 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1433 saddr_prefix_len); 1434 if (!fib_node || 1435 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1436 return NULL; 1437 1438 return fib_node->fib_entry; 1439 } 1440 1441 static struct mlxsw_sp_ipip_entry * 1442 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1443 enum mlxsw_sp_ipip_type ipipt, 1444 struct net_device *ol_dev) 1445 { 1446 struct mlxsw_sp_ipip_entry *ipip_entry; 1447 1448 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1449 if (IS_ERR(ipip_entry)) 1450 return ipip_entry; 1451 1452 list_add_tail(&ipip_entry->ipip_list_node, 1453 &mlxsw_sp->router->ipip_list); 1454 1455 return ipip_entry; 1456 } 1457 1458 static void 1459 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1460 struct mlxsw_sp_ipip_entry *ipip_entry) 1461 { 1462 list_del(&ipip_entry->ipip_list_node); 1463 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1464 } 1465 1466 static bool 1467 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1468 const struct net_device *ul_dev, 1469 enum mlxsw_sp_l3proto ul_proto, 1470 union mlxsw_sp_l3addr ul_dip, 1471 struct mlxsw_sp_ipip_entry *ipip_entry) 1472 { 1473 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1474 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1475 1476 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1477 return false; 1478 1479 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1480 ul_tb_id, ipip_entry); 1481 } 1482 1483 /* Given decap parameters, find the corresponding IPIP entry. */ 1484 static struct mlxsw_sp_ipip_entry * 1485 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1486 enum mlxsw_sp_l3proto ul_proto, 1487 union mlxsw_sp_l3addr ul_dip) 1488 { 1489 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1490 struct net_device *ul_dev; 1491 1492 rcu_read_lock(); 1493 1494 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1495 if (!ul_dev) 1496 goto out_unlock; 1497 1498 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1499 ipip_list_node) 1500 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1501 ul_proto, ul_dip, 1502 ipip_entry)) 1503 goto out_unlock; 1504 1505 rcu_read_unlock(); 1506 1507 return NULL; 1508 1509 out_unlock: 1510 rcu_read_unlock(); 1511 return ipip_entry; 1512 } 1513 1514 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1515 const struct net_device *dev, 1516 enum mlxsw_sp_ipip_type *p_type) 1517 { 1518 struct mlxsw_sp_router *router = mlxsw_sp->router; 1519 const struct mlxsw_sp_ipip_ops *ipip_ops; 1520 enum mlxsw_sp_ipip_type ipipt; 1521 1522 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1523 ipip_ops = router->ipip_ops_arr[ipipt]; 1524 if (dev->type == ipip_ops->dev_type) { 1525 if (p_type) 1526 *p_type = ipipt; 1527 return true; 1528 } 1529 } 1530 return false; 1531 } 1532 1533 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1534 const struct net_device *dev) 1535 { 1536 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1537 } 1538 1539 static struct mlxsw_sp_ipip_entry * 1540 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1541 const struct net_device *ol_dev) 1542 { 1543 struct mlxsw_sp_ipip_entry *ipip_entry; 1544 1545 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1546 ipip_list_node) 1547 if (ipip_entry->ol_dev == ol_dev) 1548 return ipip_entry; 1549 1550 return NULL; 1551 } 1552 1553 static struct mlxsw_sp_ipip_entry * 1554 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1555 const struct net_device *ul_dev, 1556 struct mlxsw_sp_ipip_entry *start) 1557 { 1558 struct mlxsw_sp_ipip_entry *ipip_entry; 1559 1560 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1561 ipip_list_node); 1562 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1563 ipip_list_node) { 1564 struct net_device *ol_dev = ipip_entry->ol_dev; 1565 struct net_device *ipip_ul_dev; 1566 1567 rcu_read_lock(); 1568 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1569 rcu_read_unlock(); 1570 1571 if (ipip_ul_dev == ul_dev) 1572 return ipip_entry; 1573 } 1574 1575 return NULL; 1576 } 1577 1578 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1579 const struct net_device *dev) 1580 { 1581 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1582 } 1583 1584 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1585 const struct net_device *ol_dev, 1586 enum mlxsw_sp_ipip_type ipipt) 1587 { 1588 const struct mlxsw_sp_ipip_ops *ops 1589 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1590 1591 return ops->can_offload(mlxsw_sp, ol_dev); 1592 } 1593 1594 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1595 struct net_device *ol_dev) 1596 { 1597 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1598 struct mlxsw_sp_ipip_entry *ipip_entry; 1599 enum mlxsw_sp_l3proto ul_proto; 1600 union mlxsw_sp_l3addr saddr; 1601 u32 ul_tb_id; 1602 1603 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1604 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1605 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1606 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1607 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1608 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1609 saddr, ul_tb_id, 1610 NULL)) { 1611 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1612 ol_dev); 1613 if (IS_ERR(ipip_entry)) 1614 return PTR_ERR(ipip_entry); 1615 } 1616 } 1617 1618 return 0; 1619 } 1620 1621 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1622 struct net_device *ol_dev) 1623 { 1624 struct mlxsw_sp_ipip_entry *ipip_entry; 1625 1626 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1627 if (ipip_entry) 1628 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1629 } 1630 1631 static void 1632 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1633 struct mlxsw_sp_ipip_entry *ipip_entry) 1634 { 1635 struct mlxsw_sp_fib_entry *decap_fib_entry; 1636 1637 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1638 if (decap_fib_entry) 1639 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1640 decap_fib_entry); 1641 } 1642 1643 static int 1644 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1645 u16 ul_rif_id, bool enable) 1646 { 1647 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1648 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1649 struct mlxsw_sp_rif *rif = &lb_rif->common; 1650 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1651 char ritr_pl[MLXSW_REG_RITR_LEN]; 1652 struct in6_addr *saddr6; 1653 u32 saddr4; 1654 1655 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1656 switch (lb_cf.ul_protocol) { 1657 case MLXSW_SP_L3_PROTO_IPV4: 1658 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1659 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1660 rif->rif_index, rif->vr_id, rif->dev->mtu); 1661 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1662 ipip_options, ul_vr_id, 1663 ul_rif_id, saddr4, 1664 lb_cf.okey); 1665 break; 1666 1667 case MLXSW_SP_L3_PROTO_IPV6: 1668 saddr6 = &lb_cf.saddr.addr6; 1669 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1670 rif->rif_index, rif->vr_id, rif->dev->mtu); 1671 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1672 ipip_options, ul_vr_id, 1673 ul_rif_id, saddr6, 1674 lb_cf.okey); 1675 break; 1676 } 1677 1678 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1679 } 1680 1681 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1682 struct net_device *ol_dev) 1683 { 1684 struct mlxsw_sp_ipip_entry *ipip_entry; 1685 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1686 int err = 0; 1687 1688 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1689 if (ipip_entry) { 1690 lb_rif = ipip_entry->ol_lb; 1691 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1692 lb_rif->ul_rif_id, true); 1693 if (err) 1694 goto out; 1695 lb_rif->common.mtu = ol_dev->mtu; 1696 } 1697 1698 out: 1699 return err; 1700 } 1701 1702 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1703 struct net_device *ol_dev) 1704 { 1705 struct mlxsw_sp_ipip_entry *ipip_entry; 1706 1707 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1708 if (ipip_entry) 1709 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1710 } 1711 1712 static void 1713 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1714 struct mlxsw_sp_ipip_entry *ipip_entry) 1715 { 1716 if (ipip_entry->decap_fib_entry) 1717 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1718 } 1719 1720 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1721 struct net_device *ol_dev) 1722 { 1723 struct mlxsw_sp_ipip_entry *ipip_entry; 1724 1725 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1726 if (ipip_entry) 1727 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1728 } 1729 1730 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1731 struct mlxsw_sp_rif *old_rif, 1732 struct mlxsw_sp_rif *new_rif); 1733 static int 1734 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1735 struct mlxsw_sp_ipip_entry *ipip_entry, 1736 bool keep_encap, 1737 struct netlink_ext_ack *extack) 1738 { 1739 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1740 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1741 1742 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1743 ipip_entry->ipipt, 1744 ipip_entry->ol_dev, 1745 extack); 1746 if (IS_ERR(new_lb_rif)) 1747 return PTR_ERR(new_lb_rif); 1748 ipip_entry->ol_lb = new_lb_rif; 1749 1750 if (keep_encap) 1751 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1752 &new_lb_rif->common); 1753 1754 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1755 1756 return 0; 1757 } 1758 1759 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1760 struct mlxsw_sp_rif *rif); 1761 1762 /** 1763 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1764 * @mlxsw_sp: mlxsw_sp. 1765 * @ipip_entry: IPIP entry. 1766 * @recreate_loopback: Recreates the associated loopback RIF. 1767 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1768 * relevant when recreate_loopback is true. 1769 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1770 * is only relevant when recreate_loopback is false. 1771 * @extack: extack. 1772 * 1773 * Return: Non-zero value on failure. 1774 */ 1775 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1776 struct mlxsw_sp_ipip_entry *ipip_entry, 1777 bool recreate_loopback, 1778 bool keep_encap, 1779 bool update_nexthops, 1780 struct netlink_ext_ack *extack) 1781 { 1782 int err; 1783 1784 /* RIFs can't be edited, so to update loopback, we need to destroy and 1785 * recreate it. That creates a window of opportunity where RALUE and 1786 * RATR registers end up referencing a RIF that's already gone. RATRs 1787 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1788 * of RALUE, demote the decap route back. 1789 */ 1790 if (ipip_entry->decap_fib_entry) 1791 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1792 1793 if (recreate_loopback) { 1794 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1795 keep_encap, extack); 1796 if (err) 1797 return err; 1798 } else if (update_nexthops) { 1799 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1800 &ipip_entry->ol_lb->common); 1801 } 1802 1803 if (ipip_entry->ol_dev->flags & IFF_UP) 1804 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1805 1806 return 0; 1807 } 1808 1809 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1810 struct net_device *ol_dev, 1811 struct netlink_ext_ack *extack) 1812 { 1813 struct mlxsw_sp_ipip_entry *ipip_entry = 1814 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1815 1816 if (!ipip_entry) 1817 return 0; 1818 1819 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1820 true, false, false, extack); 1821 } 1822 1823 static int 1824 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1825 struct mlxsw_sp_ipip_entry *ipip_entry, 1826 struct net_device *ul_dev, 1827 bool *demote_this, 1828 struct netlink_ext_ack *extack) 1829 { 1830 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1831 enum mlxsw_sp_l3proto ul_proto; 1832 union mlxsw_sp_l3addr saddr; 1833 1834 /* Moving underlay to a different VRF might cause local address 1835 * conflict, and the conflicting tunnels need to be demoted. 1836 */ 1837 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1838 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1839 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1840 saddr, ul_tb_id, 1841 ipip_entry)) { 1842 *demote_this = true; 1843 return 0; 1844 } 1845 1846 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1847 true, true, false, extack); 1848 } 1849 1850 static int 1851 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1852 struct mlxsw_sp_ipip_entry *ipip_entry, 1853 struct net_device *ul_dev) 1854 { 1855 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1856 false, false, true, NULL); 1857 } 1858 1859 static int 1860 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1861 struct mlxsw_sp_ipip_entry *ipip_entry, 1862 struct net_device *ul_dev) 1863 { 1864 /* A down underlay device causes encapsulated packets to not be 1865 * forwarded, but decap still works. So refresh next hops without 1866 * touching anything else. 1867 */ 1868 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1869 false, false, true, NULL); 1870 } 1871 1872 static int 1873 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1874 struct net_device *ol_dev, 1875 struct netlink_ext_ack *extack) 1876 { 1877 const struct mlxsw_sp_ipip_ops *ipip_ops; 1878 struct mlxsw_sp_ipip_entry *ipip_entry; 1879 int err; 1880 1881 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1882 if (!ipip_entry) 1883 /* A change might make a tunnel eligible for offloading, but 1884 * that is currently not implemented. What falls to slow path 1885 * stays there. 1886 */ 1887 return 0; 1888 1889 /* A change might make a tunnel not eligible for offloading. */ 1890 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1891 ipip_entry->ipipt)) { 1892 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1893 return 0; 1894 } 1895 1896 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1897 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1898 return err; 1899 } 1900 1901 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1902 struct mlxsw_sp_ipip_entry *ipip_entry) 1903 { 1904 struct net_device *ol_dev = ipip_entry->ol_dev; 1905 1906 if (ol_dev->flags & IFF_UP) 1907 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1908 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1909 } 1910 1911 /* The configuration where several tunnels have the same local address in the 1912 * same underlay table needs special treatment in the HW. That is currently not 1913 * implemented in the driver. This function finds and demotes the first tunnel 1914 * with a given source address, except the one passed in in the argument 1915 * `except'. 1916 */ 1917 bool 1918 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1919 enum mlxsw_sp_l3proto ul_proto, 1920 union mlxsw_sp_l3addr saddr, 1921 u32 ul_tb_id, 1922 const struct mlxsw_sp_ipip_entry *except) 1923 { 1924 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1925 1926 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1927 ipip_list_node) { 1928 if (ipip_entry != except && 1929 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1930 ul_tb_id, ipip_entry)) { 1931 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1932 return true; 1933 } 1934 } 1935 1936 return false; 1937 } 1938 1939 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1940 struct net_device *ul_dev) 1941 { 1942 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1943 1944 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1945 ipip_list_node) { 1946 struct net_device *ol_dev = ipip_entry->ol_dev; 1947 struct net_device *ipip_ul_dev; 1948 1949 rcu_read_lock(); 1950 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1951 rcu_read_unlock(); 1952 if (ipip_ul_dev == ul_dev) 1953 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1954 } 1955 } 1956 1957 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1958 struct net_device *ol_dev, 1959 unsigned long event, 1960 struct netdev_notifier_info *info) 1961 { 1962 struct netdev_notifier_changeupper_info *chup; 1963 struct netlink_ext_ack *extack; 1964 int err = 0; 1965 1966 switch (event) { 1967 case NETDEV_REGISTER: 1968 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1969 break; 1970 case NETDEV_UNREGISTER: 1971 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1972 break; 1973 case NETDEV_UP: 1974 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1975 break; 1976 case NETDEV_DOWN: 1977 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1978 break; 1979 case NETDEV_CHANGEUPPER: 1980 chup = container_of(info, typeof(*chup), info); 1981 extack = info->extack; 1982 if (netif_is_l3_master(chup->upper_dev)) 1983 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1984 ol_dev, 1985 extack); 1986 break; 1987 case NETDEV_CHANGE: 1988 extack = info->extack; 1989 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1990 ol_dev, extack); 1991 break; 1992 case NETDEV_CHANGEMTU: 1993 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1994 break; 1995 } 1996 return err; 1997 } 1998 1999 static int 2000 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2001 struct mlxsw_sp_ipip_entry *ipip_entry, 2002 struct net_device *ul_dev, 2003 bool *demote_this, 2004 unsigned long event, 2005 struct netdev_notifier_info *info) 2006 { 2007 struct netdev_notifier_changeupper_info *chup; 2008 struct netlink_ext_ack *extack; 2009 2010 switch (event) { 2011 case NETDEV_CHANGEUPPER: 2012 chup = container_of(info, typeof(*chup), info); 2013 extack = info->extack; 2014 if (netif_is_l3_master(chup->upper_dev)) 2015 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2016 ipip_entry, 2017 ul_dev, 2018 demote_this, 2019 extack); 2020 break; 2021 2022 case NETDEV_UP: 2023 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2024 ul_dev); 2025 case NETDEV_DOWN: 2026 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2027 ipip_entry, 2028 ul_dev); 2029 } 2030 return 0; 2031 } 2032 2033 static int 2034 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2035 struct net_device *ul_dev, 2036 unsigned long event, 2037 struct netdev_notifier_info *info) 2038 { 2039 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2040 int err; 2041 2042 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2043 ul_dev, 2044 ipip_entry))) { 2045 struct mlxsw_sp_ipip_entry *prev; 2046 bool demote_this = false; 2047 2048 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2049 ul_dev, &demote_this, 2050 event, info); 2051 if (err) { 2052 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2053 ul_dev); 2054 return err; 2055 } 2056 2057 if (demote_this) { 2058 if (list_is_first(&ipip_entry->ipip_list_node, 2059 &mlxsw_sp->router->ipip_list)) 2060 prev = NULL; 2061 else 2062 /* This can't be cached from previous iteration, 2063 * because that entry could be gone now. 2064 */ 2065 prev = list_prev_entry(ipip_entry, 2066 ipip_list_node); 2067 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2068 ipip_entry = prev; 2069 } 2070 } 2071 2072 return 0; 2073 } 2074 2075 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2076 enum mlxsw_sp_l3proto ul_proto, 2077 const union mlxsw_sp_l3addr *ul_sip, 2078 u32 tunnel_index) 2079 { 2080 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2081 struct mlxsw_sp_router *router = mlxsw_sp->router; 2082 struct mlxsw_sp_fib_entry *fib_entry; 2083 int err = 0; 2084 2085 mutex_lock(&mlxsw_sp->router->lock); 2086 2087 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2088 err = -EINVAL; 2089 goto out; 2090 } 2091 2092 router->nve_decap_config.ul_tb_id = ul_tb_id; 2093 router->nve_decap_config.tunnel_index = tunnel_index; 2094 router->nve_decap_config.ul_proto = ul_proto; 2095 router->nve_decap_config.ul_sip = *ul_sip; 2096 router->nve_decap_config.valid = true; 2097 2098 /* It is valid to create a tunnel with a local IP and only later 2099 * assign this IP address to a local interface 2100 */ 2101 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2102 ul_proto, ul_sip, 2103 type); 2104 if (!fib_entry) 2105 goto out; 2106 2107 fib_entry->decap.tunnel_index = tunnel_index; 2108 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2109 2110 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2111 if (err) 2112 goto err_fib_entry_update; 2113 2114 goto out; 2115 2116 err_fib_entry_update: 2117 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2118 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2119 out: 2120 mutex_unlock(&mlxsw_sp->router->lock); 2121 return err; 2122 } 2123 2124 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2125 enum mlxsw_sp_l3proto ul_proto, 2126 const union mlxsw_sp_l3addr *ul_sip) 2127 { 2128 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2129 struct mlxsw_sp_router *router = mlxsw_sp->router; 2130 struct mlxsw_sp_fib_entry *fib_entry; 2131 2132 mutex_lock(&mlxsw_sp->router->lock); 2133 2134 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2135 goto out; 2136 2137 router->nve_decap_config.valid = false; 2138 2139 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2140 ul_proto, ul_sip, 2141 type); 2142 if (!fib_entry) 2143 goto out; 2144 2145 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2146 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2147 out: 2148 mutex_unlock(&mlxsw_sp->router->lock); 2149 } 2150 2151 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2152 u32 ul_tb_id, 2153 enum mlxsw_sp_l3proto ul_proto, 2154 const union mlxsw_sp_l3addr *ul_sip) 2155 { 2156 struct mlxsw_sp_router *router = mlxsw_sp->router; 2157 2158 return router->nve_decap_config.valid && 2159 router->nve_decap_config.ul_tb_id == ul_tb_id && 2160 router->nve_decap_config.ul_proto == ul_proto && 2161 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2162 sizeof(*ul_sip)); 2163 } 2164 2165 struct mlxsw_sp_neigh_key { 2166 struct neighbour *n; 2167 }; 2168 2169 struct mlxsw_sp_neigh_entry { 2170 struct list_head rif_list_node; 2171 struct rhash_head ht_node; 2172 struct mlxsw_sp_neigh_key key; 2173 u16 rif; 2174 bool connected; 2175 unsigned char ha[ETH_ALEN]; 2176 struct list_head nexthop_list; /* list of nexthops using 2177 * this neigh entry 2178 */ 2179 struct list_head nexthop_neighs_list_node; 2180 unsigned int counter_index; 2181 bool counter_valid; 2182 }; 2183 2184 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2185 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2186 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2187 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2188 }; 2189 2190 struct mlxsw_sp_neigh_entry * 2191 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2192 struct mlxsw_sp_neigh_entry *neigh_entry) 2193 { 2194 if (!neigh_entry) { 2195 if (list_empty(&rif->neigh_list)) 2196 return NULL; 2197 else 2198 return list_first_entry(&rif->neigh_list, 2199 typeof(*neigh_entry), 2200 rif_list_node); 2201 } 2202 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2203 return NULL; 2204 return list_next_entry(neigh_entry, rif_list_node); 2205 } 2206 2207 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2208 { 2209 return neigh_entry->key.n->tbl->family; 2210 } 2211 2212 unsigned char * 2213 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2214 { 2215 return neigh_entry->ha; 2216 } 2217 2218 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2219 { 2220 struct neighbour *n; 2221 2222 n = neigh_entry->key.n; 2223 return ntohl(*((__be32 *) n->primary_key)); 2224 } 2225 2226 struct in6_addr * 2227 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2228 { 2229 struct neighbour *n; 2230 2231 n = neigh_entry->key.n; 2232 return (struct in6_addr *) &n->primary_key; 2233 } 2234 2235 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2236 struct mlxsw_sp_neigh_entry *neigh_entry, 2237 u64 *p_counter) 2238 { 2239 if (!neigh_entry->counter_valid) 2240 return -EINVAL; 2241 2242 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2243 p_counter, NULL); 2244 } 2245 2246 static struct mlxsw_sp_neigh_entry * 2247 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2248 u16 rif) 2249 { 2250 struct mlxsw_sp_neigh_entry *neigh_entry; 2251 2252 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2253 if (!neigh_entry) 2254 return NULL; 2255 2256 neigh_entry->key.n = n; 2257 neigh_entry->rif = rif; 2258 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2259 2260 return neigh_entry; 2261 } 2262 2263 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2264 { 2265 kfree(neigh_entry); 2266 } 2267 2268 static int 2269 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2270 struct mlxsw_sp_neigh_entry *neigh_entry) 2271 { 2272 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2273 &neigh_entry->ht_node, 2274 mlxsw_sp_neigh_ht_params); 2275 } 2276 2277 static void 2278 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2279 struct mlxsw_sp_neigh_entry *neigh_entry) 2280 { 2281 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2282 &neigh_entry->ht_node, 2283 mlxsw_sp_neigh_ht_params); 2284 } 2285 2286 static bool 2287 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2288 struct mlxsw_sp_neigh_entry *neigh_entry) 2289 { 2290 struct devlink *devlink; 2291 const char *table_name; 2292 2293 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2294 case AF_INET: 2295 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2296 break; 2297 case AF_INET6: 2298 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2299 break; 2300 default: 2301 WARN_ON(1); 2302 return false; 2303 } 2304 2305 devlink = priv_to_devlink(mlxsw_sp->core); 2306 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2307 } 2308 2309 static void 2310 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2311 struct mlxsw_sp_neigh_entry *neigh_entry) 2312 { 2313 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2314 return; 2315 2316 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2317 return; 2318 2319 neigh_entry->counter_valid = true; 2320 } 2321 2322 static void 2323 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2324 struct mlxsw_sp_neigh_entry *neigh_entry) 2325 { 2326 if (!neigh_entry->counter_valid) 2327 return; 2328 mlxsw_sp_flow_counter_free(mlxsw_sp, 2329 neigh_entry->counter_index); 2330 neigh_entry->counter_valid = false; 2331 } 2332 2333 static struct mlxsw_sp_neigh_entry * 2334 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2335 { 2336 struct mlxsw_sp_neigh_entry *neigh_entry; 2337 struct mlxsw_sp_rif *rif; 2338 int err; 2339 2340 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2341 if (!rif) 2342 return ERR_PTR(-EINVAL); 2343 2344 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2345 if (!neigh_entry) 2346 return ERR_PTR(-ENOMEM); 2347 2348 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2349 if (err) 2350 goto err_neigh_entry_insert; 2351 2352 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2353 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2354 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2355 2356 return neigh_entry; 2357 2358 err_neigh_entry_insert: 2359 mlxsw_sp_neigh_entry_free(neigh_entry); 2360 return ERR_PTR(err); 2361 } 2362 2363 static void 2364 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2365 struct mlxsw_sp_neigh_entry *neigh_entry) 2366 { 2367 list_del(&neigh_entry->rif_list_node); 2368 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2369 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2370 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2371 mlxsw_sp_neigh_entry_free(neigh_entry); 2372 } 2373 2374 static struct mlxsw_sp_neigh_entry * 2375 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2376 { 2377 struct mlxsw_sp_neigh_key key; 2378 2379 key.n = n; 2380 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2381 &key, mlxsw_sp_neigh_ht_params); 2382 } 2383 2384 static void 2385 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2386 { 2387 unsigned long interval; 2388 2389 #if IS_ENABLED(CONFIG_IPV6) 2390 interval = min_t(unsigned long, 2391 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2392 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2393 #else 2394 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2395 #endif 2396 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2397 } 2398 2399 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2400 char *rauhtd_pl, 2401 int ent_index) 2402 { 2403 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2404 struct net_device *dev; 2405 struct neighbour *n; 2406 __be32 dipn; 2407 u32 dip; 2408 u16 rif; 2409 2410 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2411 2412 if (WARN_ON_ONCE(rif >= max_rifs)) 2413 return; 2414 if (!mlxsw_sp->router->rifs[rif]) { 2415 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2416 return; 2417 } 2418 2419 dipn = htonl(dip); 2420 dev = mlxsw_sp->router->rifs[rif]->dev; 2421 n = neigh_lookup(&arp_tbl, &dipn, dev); 2422 if (!n) 2423 return; 2424 2425 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2426 neigh_event_send(n, NULL); 2427 neigh_release(n); 2428 } 2429 2430 #if IS_ENABLED(CONFIG_IPV6) 2431 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2432 char *rauhtd_pl, 2433 int rec_index) 2434 { 2435 struct net_device *dev; 2436 struct neighbour *n; 2437 struct in6_addr dip; 2438 u16 rif; 2439 2440 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2441 (char *) &dip); 2442 2443 if (!mlxsw_sp->router->rifs[rif]) { 2444 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2445 return; 2446 } 2447 2448 dev = mlxsw_sp->router->rifs[rif]->dev; 2449 n = neigh_lookup(&nd_tbl, &dip, dev); 2450 if (!n) 2451 return; 2452 2453 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2454 neigh_event_send(n, NULL); 2455 neigh_release(n); 2456 } 2457 #else 2458 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2459 char *rauhtd_pl, 2460 int rec_index) 2461 { 2462 } 2463 #endif 2464 2465 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2466 char *rauhtd_pl, 2467 int rec_index) 2468 { 2469 u8 num_entries; 2470 int i; 2471 2472 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2473 rec_index); 2474 /* Hardware starts counting at 0, so add 1. */ 2475 num_entries++; 2476 2477 /* Each record consists of several neighbour entries. */ 2478 for (i = 0; i < num_entries; i++) { 2479 int ent_index; 2480 2481 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2482 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2483 ent_index); 2484 } 2485 2486 } 2487 2488 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2489 char *rauhtd_pl, 2490 int rec_index) 2491 { 2492 /* One record contains one entry. */ 2493 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2494 rec_index); 2495 } 2496 2497 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2498 char *rauhtd_pl, int rec_index) 2499 { 2500 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2501 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2502 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2503 rec_index); 2504 break; 2505 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2506 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2507 rec_index); 2508 break; 2509 } 2510 } 2511 2512 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2513 { 2514 u8 num_rec, last_rec_index, num_entries; 2515 2516 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2517 last_rec_index = num_rec - 1; 2518 2519 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2520 return false; 2521 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2522 MLXSW_REG_RAUHTD_TYPE_IPV6) 2523 return true; 2524 2525 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2526 last_rec_index); 2527 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2528 return true; 2529 return false; 2530 } 2531 2532 static int 2533 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2534 char *rauhtd_pl, 2535 enum mlxsw_reg_rauhtd_type type) 2536 { 2537 int i, num_rec; 2538 int err; 2539 2540 /* Ensure the RIF we read from the device does not change mid-dump. */ 2541 mutex_lock(&mlxsw_sp->router->lock); 2542 do { 2543 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2544 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2545 rauhtd_pl); 2546 if (err) { 2547 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2548 break; 2549 } 2550 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2551 for (i = 0; i < num_rec; i++) 2552 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2553 i); 2554 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2555 mutex_unlock(&mlxsw_sp->router->lock); 2556 2557 return err; 2558 } 2559 2560 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2561 { 2562 enum mlxsw_reg_rauhtd_type type; 2563 char *rauhtd_pl; 2564 int err; 2565 2566 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2567 return 0; 2568 2569 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2570 if (!rauhtd_pl) 2571 return -ENOMEM; 2572 2573 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2574 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2575 if (err) 2576 goto out; 2577 2578 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2579 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2580 out: 2581 kfree(rauhtd_pl); 2582 return err; 2583 } 2584 2585 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2586 { 2587 struct mlxsw_sp_neigh_entry *neigh_entry; 2588 2589 mutex_lock(&mlxsw_sp->router->lock); 2590 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2591 nexthop_neighs_list_node) 2592 /* If this neigh have nexthops, make the kernel think this neigh 2593 * is active regardless of the traffic. 2594 */ 2595 neigh_event_send(neigh_entry->key.n, NULL); 2596 mutex_unlock(&mlxsw_sp->router->lock); 2597 } 2598 2599 static void 2600 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2601 { 2602 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2603 2604 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2605 msecs_to_jiffies(interval)); 2606 } 2607 2608 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2609 { 2610 struct mlxsw_sp_router *router; 2611 int err; 2612 2613 router = container_of(work, struct mlxsw_sp_router, 2614 neighs_update.dw.work); 2615 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2616 if (err) 2617 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2618 2619 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2620 2621 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2622 } 2623 2624 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2625 { 2626 struct mlxsw_sp_neigh_entry *neigh_entry; 2627 struct mlxsw_sp_router *router; 2628 2629 router = container_of(work, struct mlxsw_sp_router, 2630 nexthop_probe_dw.work); 2631 /* Iterate over nexthop neighbours, find those who are unresolved and 2632 * send arp on them. This solves the chicken-egg problem when 2633 * the nexthop wouldn't get offloaded until the neighbor is resolved 2634 * but it wouldn't get resolved ever in case traffic is flowing in HW 2635 * using different nexthop. 2636 */ 2637 mutex_lock(&router->lock); 2638 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2639 nexthop_neighs_list_node) 2640 if (!neigh_entry->connected) 2641 neigh_event_send(neigh_entry->key.n, NULL); 2642 mutex_unlock(&router->lock); 2643 2644 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2645 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2646 } 2647 2648 static void 2649 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2650 struct mlxsw_sp_neigh_entry *neigh_entry, 2651 bool removing, bool dead); 2652 2653 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2654 { 2655 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2656 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2657 } 2658 2659 static int 2660 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2661 struct mlxsw_sp_neigh_entry *neigh_entry, 2662 enum mlxsw_reg_rauht_op op) 2663 { 2664 struct neighbour *n = neigh_entry->key.n; 2665 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2666 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2667 2668 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2669 dip); 2670 if (neigh_entry->counter_valid) 2671 mlxsw_reg_rauht_pack_counter(rauht_pl, 2672 neigh_entry->counter_index); 2673 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2674 } 2675 2676 static int 2677 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2678 struct mlxsw_sp_neigh_entry *neigh_entry, 2679 enum mlxsw_reg_rauht_op op) 2680 { 2681 struct neighbour *n = neigh_entry->key.n; 2682 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2683 const char *dip = n->primary_key; 2684 2685 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2686 dip); 2687 if (neigh_entry->counter_valid) 2688 mlxsw_reg_rauht_pack_counter(rauht_pl, 2689 neigh_entry->counter_index); 2690 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2691 } 2692 2693 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2694 { 2695 struct neighbour *n = neigh_entry->key.n; 2696 2697 /* Packets with a link-local destination address are trapped 2698 * after LPM lookup and never reach the neighbour table, so 2699 * there is no need to program such neighbours to the device. 2700 */ 2701 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2702 IPV6_ADDR_LINKLOCAL) 2703 return true; 2704 return false; 2705 } 2706 2707 static void 2708 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2709 struct mlxsw_sp_neigh_entry *neigh_entry, 2710 bool adding) 2711 { 2712 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2713 int err; 2714 2715 if (!adding && !neigh_entry->connected) 2716 return; 2717 neigh_entry->connected = adding; 2718 if (neigh_entry->key.n->tbl->family == AF_INET) { 2719 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2720 op); 2721 if (err) 2722 return; 2723 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2724 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2725 return; 2726 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2727 op); 2728 if (err) 2729 return; 2730 } else { 2731 WARN_ON_ONCE(1); 2732 return; 2733 } 2734 2735 if (adding) 2736 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2737 else 2738 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2739 } 2740 2741 void 2742 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2743 struct mlxsw_sp_neigh_entry *neigh_entry, 2744 bool adding) 2745 { 2746 if (adding) 2747 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2748 else 2749 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2750 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2751 } 2752 2753 struct mlxsw_sp_netevent_work { 2754 struct work_struct work; 2755 struct mlxsw_sp *mlxsw_sp; 2756 struct neighbour *n; 2757 }; 2758 2759 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2760 { 2761 struct mlxsw_sp_netevent_work *net_work = 2762 container_of(work, struct mlxsw_sp_netevent_work, work); 2763 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2764 struct mlxsw_sp_neigh_entry *neigh_entry; 2765 struct neighbour *n = net_work->n; 2766 unsigned char ha[ETH_ALEN]; 2767 bool entry_connected; 2768 u8 nud_state, dead; 2769 2770 /* If these parameters are changed after we release the lock, 2771 * then we are guaranteed to receive another event letting us 2772 * know about it. 2773 */ 2774 read_lock_bh(&n->lock); 2775 memcpy(ha, n->ha, ETH_ALEN); 2776 nud_state = n->nud_state; 2777 dead = n->dead; 2778 read_unlock_bh(&n->lock); 2779 2780 mutex_lock(&mlxsw_sp->router->lock); 2781 mlxsw_sp_span_respin(mlxsw_sp); 2782 2783 entry_connected = nud_state & NUD_VALID && !dead; 2784 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2785 if (!entry_connected && !neigh_entry) 2786 goto out; 2787 if (!neigh_entry) { 2788 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2789 if (IS_ERR(neigh_entry)) 2790 goto out; 2791 } 2792 2793 if (neigh_entry->connected && entry_connected && 2794 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2795 goto out; 2796 2797 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2798 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2799 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2800 dead); 2801 2802 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2803 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2804 2805 out: 2806 mutex_unlock(&mlxsw_sp->router->lock); 2807 neigh_release(n); 2808 kfree(net_work); 2809 } 2810 2811 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2812 2813 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2814 { 2815 struct mlxsw_sp_netevent_work *net_work = 2816 container_of(work, struct mlxsw_sp_netevent_work, work); 2817 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2818 2819 mlxsw_sp_mp_hash_init(mlxsw_sp); 2820 kfree(net_work); 2821 } 2822 2823 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2824 2825 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2826 { 2827 struct mlxsw_sp_netevent_work *net_work = 2828 container_of(work, struct mlxsw_sp_netevent_work, work); 2829 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2830 2831 __mlxsw_sp_router_init(mlxsw_sp); 2832 kfree(net_work); 2833 } 2834 2835 static int mlxsw_sp_router_schedule_work(struct net *net, 2836 struct notifier_block *nb, 2837 void (*cb)(struct work_struct *)) 2838 { 2839 struct mlxsw_sp_netevent_work *net_work; 2840 struct mlxsw_sp_router *router; 2841 2842 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2843 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2844 return NOTIFY_DONE; 2845 2846 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2847 if (!net_work) 2848 return NOTIFY_BAD; 2849 2850 INIT_WORK(&net_work->work, cb); 2851 net_work->mlxsw_sp = router->mlxsw_sp; 2852 mlxsw_core_schedule_work(&net_work->work); 2853 return NOTIFY_DONE; 2854 } 2855 2856 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2857 unsigned long event, void *ptr) 2858 { 2859 struct mlxsw_sp_netevent_work *net_work; 2860 struct mlxsw_sp_port *mlxsw_sp_port; 2861 struct mlxsw_sp *mlxsw_sp; 2862 unsigned long interval; 2863 struct neigh_parms *p; 2864 struct neighbour *n; 2865 2866 switch (event) { 2867 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2868 p = ptr; 2869 2870 /* We don't care about changes in the default table. */ 2871 if (!p->dev || (p->tbl->family != AF_INET && 2872 p->tbl->family != AF_INET6)) 2873 return NOTIFY_DONE; 2874 2875 /* We are in atomic context and can't take RTNL mutex, 2876 * so use RCU variant to walk the device chain. 2877 */ 2878 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2879 if (!mlxsw_sp_port) 2880 return NOTIFY_DONE; 2881 2882 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2883 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2884 mlxsw_sp->router->neighs_update.interval = interval; 2885 2886 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2887 break; 2888 case NETEVENT_NEIGH_UPDATE: 2889 n = ptr; 2890 2891 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2892 return NOTIFY_DONE; 2893 2894 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2895 if (!mlxsw_sp_port) 2896 return NOTIFY_DONE; 2897 2898 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2899 if (!net_work) { 2900 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2901 return NOTIFY_BAD; 2902 } 2903 2904 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2905 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2906 net_work->n = n; 2907 2908 /* Take a reference to ensure the neighbour won't be 2909 * destructed until we drop the reference in delayed 2910 * work. 2911 */ 2912 neigh_clone(n); 2913 mlxsw_core_schedule_work(&net_work->work); 2914 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2915 break; 2916 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2917 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2918 return mlxsw_sp_router_schedule_work(ptr, nb, 2919 mlxsw_sp_router_mp_hash_event_work); 2920 2921 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2922 return mlxsw_sp_router_schedule_work(ptr, nb, 2923 mlxsw_sp_router_update_priority_work); 2924 } 2925 2926 return NOTIFY_DONE; 2927 } 2928 2929 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2930 { 2931 int err; 2932 2933 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2934 &mlxsw_sp_neigh_ht_params); 2935 if (err) 2936 return err; 2937 2938 /* Initialize the polling interval according to the default 2939 * table. 2940 */ 2941 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2942 2943 /* Create the delayed works for the activity_update */ 2944 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2945 mlxsw_sp_router_neighs_update_work); 2946 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2947 mlxsw_sp_router_probe_unresolved_nexthops); 2948 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2949 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2950 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2951 return 0; 2952 } 2953 2954 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2955 { 2956 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2957 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2958 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2959 } 2960 2961 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2962 struct mlxsw_sp_rif *rif) 2963 { 2964 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2965 2966 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2967 rif_list_node) { 2968 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2969 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2970 } 2971 } 2972 2973 enum mlxsw_sp_nexthop_type { 2974 MLXSW_SP_NEXTHOP_TYPE_ETH, 2975 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2976 }; 2977 2978 enum mlxsw_sp_nexthop_action { 2979 /* Nexthop forwards packets to an egress RIF */ 2980 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2981 /* Nexthop discards packets */ 2982 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2983 /* Nexthop traps packets */ 2984 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2985 }; 2986 2987 struct mlxsw_sp_nexthop_key { 2988 struct fib_nh *fib_nh; 2989 }; 2990 2991 struct mlxsw_sp_nexthop { 2992 struct list_head neigh_list_node; /* member of neigh entry list */ 2993 struct list_head rif_list_node; 2994 struct list_head router_list_node; 2995 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2996 * this nexthop belongs to 2997 */ 2998 struct rhash_head ht_node; 2999 struct neigh_table *neigh_tbl; 3000 struct mlxsw_sp_nexthop_key key; 3001 unsigned char gw_addr[sizeof(struct in6_addr)]; 3002 int ifindex; 3003 int nh_weight; 3004 int norm_nh_weight; 3005 int num_adj_entries; 3006 struct mlxsw_sp_rif *rif; 3007 u8 should_offload:1, /* set indicates this nexthop should be written 3008 * to the adjacency table. 3009 */ 3010 offloaded:1, /* set indicates this nexthop was written to the 3011 * adjacency table. 3012 */ 3013 update:1; /* set indicates this nexthop should be updated in the 3014 * adjacency table (f.e., its MAC changed). 3015 */ 3016 enum mlxsw_sp_nexthop_action action; 3017 enum mlxsw_sp_nexthop_type type; 3018 union { 3019 struct mlxsw_sp_neigh_entry *neigh_entry; 3020 struct mlxsw_sp_ipip_entry *ipip_entry; 3021 }; 3022 unsigned int counter_index; 3023 bool counter_valid; 3024 }; 3025 3026 enum mlxsw_sp_nexthop_group_type { 3027 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3028 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3029 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3030 }; 3031 3032 struct mlxsw_sp_nexthop_group_info { 3033 struct mlxsw_sp_nexthop_group *nh_grp; 3034 u32 adj_index; 3035 u16 ecmp_size; 3036 u16 count; 3037 int sum_norm_weight; 3038 u8 adj_index_valid:1, 3039 gateway:1, /* routes using the group use a gateway */ 3040 is_resilient:1; 3041 struct list_head list; /* member in nh_res_grp_list */ 3042 struct mlxsw_sp_nexthop nexthops[0]; 3043 #define nh_rif nexthops[0].rif 3044 }; 3045 3046 struct mlxsw_sp_nexthop_group_vr_key { 3047 u16 vr_id; 3048 enum mlxsw_sp_l3proto proto; 3049 }; 3050 3051 struct mlxsw_sp_nexthop_group_vr_entry { 3052 struct list_head list; /* member in vr_list */ 3053 struct rhash_head ht_node; /* member in vr_ht */ 3054 refcount_t ref_count; 3055 struct mlxsw_sp_nexthop_group_vr_key key; 3056 }; 3057 3058 struct mlxsw_sp_nexthop_group { 3059 struct rhash_head ht_node; 3060 struct list_head fib_list; /* list of fib entries that use this group */ 3061 union { 3062 struct { 3063 struct fib_info *fi; 3064 } ipv4; 3065 struct { 3066 u32 id; 3067 } obj; 3068 }; 3069 struct mlxsw_sp_nexthop_group_info *nhgi; 3070 struct list_head vr_list; 3071 struct rhashtable vr_ht; 3072 enum mlxsw_sp_nexthop_group_type type; 3073 bool can_destroy; 3074 }; 3075 3076 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 3077 struct mlxsw_sp_nexthop *nh) 3078 { 3079 struct devlink *devlink; 3080 3081 devlink = priv_to_devlink(mlxsw_sp->core); 3082 if (!devlink_dpipe_table_counter_enabled(devlink, 3083 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 3084 return; 3085 3086 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 3087 return; 3088 3089 nh->counter_valid = true; 3090 } 3091 3092 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3093 struct mlxsw_sp_nexthop *nh) 3094 { 3095 if (!nh->counter_valid) 3096 return; 3097 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3098 nh->counter_valid = false; 3099 } 3100 3101 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3102 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3103 { 3104 if (!nh->counter_valid) 3105 return -EINVAL; 3106 3107 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3108 p_counter, NULL); 3109 } 3110 3111 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3112 struct mlxsw_sp_nexthop *nh) 3113 { 3114 if (!nh) { 3115 if (list_empty(&router->nexthop_list)) 3116 return NULL; 3117 else 3118 return list_first_entry(&router->nexthop_list, 3119 typeof(*nh), router_list_node); 3120 } 3121 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3122 return NULL; 3123 return list_next_entry(nh, router_list_node); 3124 } 3125 3126 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3127 { 3128 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3129 } 3130 3131 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3132 { 3133 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3134 !mlxsw_sp_nexthop_is_forward(nh)) 3135 return NULL; 3136 return nh->neigh_entry->ha; 3137 } 3138 3139 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3140 u32 *p_adj_size, u32 *p_adj_hash_index) 3141 { 3142 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3143 u32 adj_hash_index = 0; 3144 int i; 3145 3146 if (!nh->offloaded || !nhgi->adj_index_valid) 3147 return -EINVAL; 3148 3149 *p_adj_index = nhgi->adj_index; 3150 *p_adj_size = nhgi->ecmp_size; 3151 3152 for (i = 0; i < nhgi->count; i++) { 3153 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3154 3155 if (nh_iter == nh) 3156 break; 3157 if (nh_iter->offloaded) 3158 adj_hash_index += nh_iter->num_adj_entries; 3159 } 3160 3161 *p_adj_hash_index = adj_hash_index; 3162 return 0; 3163 } 3164 3165 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3166 { 3167 return nh->rif; 3168 } 3169 3170 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3171 { 3172 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3173 int i; 3174 3175 for (i = 0; i < nhgi->count; i++) { 3176 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3177 3178 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3179 return true; 3180 } 3181 return false; 3182 } 3183 3184 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3185 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3186 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3187 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3188 .automatic_shrinking = true, 3189 }; 3190 3191 static struct mlxsw_sp_nexthop_group_vr_entry * 3192 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3193 const struct mlxsw_sp_fib *fib) 3194 { 3195 struct mlxsw_sp_nexthop_group_vr_key key; 3196 3197 memset(&key, 0, sizeof(key)); 3198 key.vr_id = fib->vr->id; 3199 key.proto = fib->proto; 3200 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3201 mlxsw_sp_nexthop_group_vr_ht_params); 3202 } 3203 3204 static int 3205 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3206 const struct mlxsw_sp_fib *fib) 3207 { 3208 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3209 int err; 3210 3211 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3212 if (!vr_entry) 3213 return -ENOMEM; 3214 3215 vr_entry->key.vr_id = fib->vr->id; 3216 vr_entry->key.proto = fib->proto; 3217 refcount_set(&vr_entry->ref_count, 1); 3218 3219 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3220 mlxsw_sp_nexthop_group_vr_ht_params); 3221 if (err) 3222 goto err_hashtable_insert; 3223 3224 list_add(&vr_entry->list, &nh_grp->vr_list); 3225 3226 return 0; 3227 3228 err_hashtable_insert: 3229 kfree(vr_entry); 3230 return err; 3231 } 3232 3233 static void 3234 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3235 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3236 { 3237 list_del(&vr_entry->list); 3238 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3239 mlxsw_sp_nexthop_group_vr_ht_params); 3240 kfree(vr_entry); 3241 } 3242 3243 static int 3244 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3245 const struct mlxsw_sp_fib *fib) 3246 { 3247 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3248 3249 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3250 if (vr_entry) { 3251 refcount_inc(&vr_entry->ref_count); 3252 return 0; 3253 } 3254 3255 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3256 } 3257 3258 static void 3259 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3260 const struct mlxsw_sp_fib *fib) 3261 { 3262 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3263 3264 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3265 if (WARN_ON_ONCE(!vr_entry)) 3266 return; 3267 3268 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3269 return; 3270 3271 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3272 } 3273 3274 struct mlxsw_sp_nexthop_group_cmp_arg { 3275 enum mlxsw_sp_nexthop_group_type type; 3276 union { 3277 struct fib_info *fi; 3278 struct mlxsw_sp_fib6_entry *fib6_entry; 3279 u32 id; 3280 }; 3281 }; 3282 3283 static bool 3284 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3285 const struct in6_addr *gw, int ifindex, 3286 int weight) 3287 { 3288 int i; 3289 3290 for (i = 0; i < nh_grp->nhgi->count; i++) { 3291 const struct mlxsw_sp_nexthop *nh; 3292 3293 nh = &nh_grp->nhgi->nexthops[i]; 3294 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3295 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3296 return true; 3297 } 3298 3299 return false; 3300 } 3301 3302 static bool 3303 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3304 const struct mlxsw_sp_fib6_entry *fib6_entry) 3305 { 3306 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3307 3308 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3309 return false; 3310 3311 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3312 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3313 struct in6_addr *gw; 3314 int ifindex, weight; 3315 3316 ifindex = fib6_nh->fib_nh_dev->ifindex; 3317 weight = fib6_nh->fib_nh_weight; 3318 gw = &fib6_nh->fib_nh_gw6; 3319 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3320 weight)) 3321 return false; 3322 } 3323 3324 return true; 3325 } 3326 3327 static int 3328 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3329 { 3330 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3331 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3332 3333 if (nh_grp->type != cmp_arg->type) 3334 return 1; 3335 3336 switch (cmp_arg->type) { 3337 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3338 return cmp_arg->fi != nh_grp->ipv4.fi; 3339 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3340 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3341 cmp_arg->fib6_entry); 3342 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3343 return cmp_arg->id != nh_grp->obj.id; 3344 default: 3345 WARN_ON(1); 3346 return 1; 3347 } 3348 } 3349 3350 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3351 { 3352 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3353 const struct mlxsw_sp_nexthop *nh; 3354 struct fib_info *fi; 3355 unsigned int val; 3356 int i; 3357 3358 switch (nh_grp->type) { 3359 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3360 fi = nh_grp->ipv4.fi; 3361 return jhash(&fi, sizeof(fi), seed); 3362 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3363 val = nh_grp->nhgi->count; 3364 for (i = 0; i < nh_grp->nhgi->count; i++) { 3365 nh = &nh_grp->nhgi->nexthops[i]; 3366 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3367 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3368 } 3369 return jhash(&val, sizeof(val), seed); 3370 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3371 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3372 default: 3373 WARN_ON(1); 3374 return 0; 3375 } 3376 } 3377 3378 static u32 3379 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3380 { 3381 unsigned int val = fib6_entry->nrt6; 3382 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3383 3384 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3385 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3386 struct net_device *dev = fib6_nh->fib_nh_dev; 3387 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3388 3389 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3390 val ^= jhash(gw, sizeof(*gw), seed); 3391 } 3392 3393 return jhash(&val, sizeof(val), seed); 3394 } 3395 3396 static u32 3397 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3398 { 3399 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3400 3401 switch (cmp_arg->type) { 3402 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3403 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3404 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3405 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3406 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3407 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3408 default: 3409 WARN_ON(1); 3410 return 0; 3411 } 3412 } 3413 3414 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3415 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3416 .hashfn = mlxsw_sp_nexthop_group_hash, 3417 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3418 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3419 }; 3420 3421 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3422 struct mlxsw_sp_nexthop_group *nh_grp) 3423 { 3424 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3425 !nh_grp->nhgi->gateway) 3426 return 0; 3427 3428 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3429 &nh_grp->ht_node, 3430 mlxsw_sp_nexthop_group_ht_params); 3431 } 3432 3433 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3434 struct mlxsw_sp_nexthop_group *nh_grp) 3435 { 3436 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3437 !nh_grp->nhgi->gateway) 3438 return; 3439 3440 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3441 &nh_grp->ht_node, 3442 mlxsw_sp_nexthop_group_ht_params); 3443 } 3444 3445 static struct mlxsw_sp_nexthop_group * 3446 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3447 struct fib_info *fi) 3448 { 3449 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3450 3451 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3452 cmp_arg.fi = fi; 3453 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3454 &cmp_arg, 3455 mlxsw_sp_nexthop_group_ht_params); 3456 } 3457 3458 static struct mlxsw_sp_nexthop_group * 3459 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3460 struct mlxsw_sp_fib6_entry *fib6_entry) 3461 { 3462 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3463 3464 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3465 cmp_arg.fib6_entry = fib6_entry; 3466 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3467 &cmp_arg, 3468 mlxsw_sp_nexthop_group_ht_params); 3469 } 3470 3471 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3472 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3473 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3474 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3475 }; 3476 3477 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3478 struct mlxsw_sp_nexthop *nh) 3479 { 3480 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3481 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3482 } 3483 3484 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3485 struct mlxsw_sp_nexthop *nh) 3486 { 3487 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3488 mlxsw_sp_nexthop_ht_params); 3489 } 3490 3491 static struct mlxsw_sp_nexthop * 3492 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3493 struct mlxsw_sp_nexthop_key key) 3494 { 3495 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3496 mlxsw_sp_nexthop_ht_params); 3497 } 3498 3499 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3500 enum mlxsw_sp_l3proto proto, 3501 u16 vr_id, 3502 u32 adj_index, u16 ecmp_size, 3503 u32 new_adj_index, 3504 u16 new_ecmp_size) 3505 { 3506 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3507 3508 mlxsw_reg_raleu_pack(raleu_pl, 3509 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3510 adj_index, ecmp_size, new_adj_index, 3511 new_ecmp_size); 3512 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3513 } 3514 3515 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3516 struct mlxsw_sp_nexthop_group *nh_grp, 3517 u32 old_adj_index, u16 old_ecmp_size) 3518 { 3519 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3520 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3521 int err; 3522 3523 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3524 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3525 vr_entry->key.proto, 3526 vr_entry->key.vr_id, 3527 old_adj_index, 3528 old_ecmp_size, 3529 nhgi->adj_index, 3530 nhgi->ecmp_size); 3531 if (err) 3532 goto err_mass_update_vr; 3533 } 3534 return 0; 3535 3536 err_mass_update_vr: 3537 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3538 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3539 vr_entry->key.vr_id, 3540 nhgi->adj_index, 3541 nhgi->ecmp_size, 3542 old_adj_index, old_ecmp_size); 3543 return err; 3544 } 3545 3546 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3547 u32 adj_index, 3548 struct mlxsw_sp_nexthop *nh, 3549 bool force, char *ratr_pl) 3550 { 3551 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3552 enum mlxsw_reg_ratr_op op; 3553 u16 rif_index; 3554 3555 rif_index = nh->rif ? nh->rif->rif_index : 3556 mlxsw_sp->router->lb_rif_index; 3557 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3558 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3559 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3560 adj_index, rif_index); 3561 switch (nh->action) { 3562 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3563 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3564 break; 3565 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3566 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3567 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3568 break; 3569 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3570 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3571 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3572 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3573 break; 3574 default: 3575 WARN_ON_ONCE(1); 3576 return -EINVAL; 3577 } 3578 if (nh->counter_valid) 3579 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3580 else 3581 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3582 3583 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3584 } 3585 3586 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3587 struct mlxsw_sp_nexthop *nh, bool force, 3588 char *ratr_pl) 3589 { 3590 int i; 3591 3592 for (i = 0; i < nh->num_adj_entries; i++) { 3593 int err; 3594 3595 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3596 nh, force, ratr_pl); 3597 if (err) 3598 return err; 3599 } 3600 3601 return 0; 3602 } 3603 3604 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3605 u32 adj_index, 3606 struct mlxsw_sp_nexthop *nh, 3607 bool force, char *ratr_pl) 3608 { 3609 const struct mlxsw_sp_ipip_ops *ipip_ops; 3610 3611 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3612 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3613 force, ratr_pl); 3614 } 3615 3616 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3617 u32 adj_index, 3618 struct mlxsw_sp_nexthop *nh, bool force, 3619 char *ratr_pl) 3620 { 3621 int i; 3622 3623 for (i = 0; i < nh->num_adj_entries; i++) { 3624 int err; 3625 3626 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3627 nh, force, ratr_pl); 3628 if (err) 3629 return err; 3630 } 3631 3632 return 0; 3633 } 3634 3635 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3636 struct mlxsw_sp_nexthop *nh, bool force, 3637 char *ratr_pl) 3638 { 3639 /* When action is discard or trap, the nexthop must be 3640 * programmed as an Ethernet nexthop. 3641 */ 3642 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3643 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3644 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3645 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3646 force, ratr_pl); 3647 else 3648 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3649 force, ratr_pl); 3650 } 3651 3652 static int 3653 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3654 struct mlxsw_sp_nexthop_group_info *nhgi, 3655 bool reallocate) 3656 { 3657 char ratr_pl[MLXSW_REG_RATR_LEN]; 3658 u32 adj_index = nhgi->adj_index; /* base */ 3659 struct mlxsw_sp_nexthop *nh; 3660 int i; 3661 3662 for (i = 0; i < nhgi->count; i++) { 3663 nh = &nhgi->nexthops[i]; 3664 3665 if (!nh->should_offload) { 3666 nh->offloaded = 0; 3667 continue; 3668 } 3669 3670 if (nh->update || reallocate) { 3671 int err = 0; 3672 3673 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3674 true, ratr_pl); 3675 if (err) 3676 return err; 3677 nh->update = 0; 3678 nh->offloaded = 1; 3679 } 3680 adj_index += nh->num_adj_entries; 3681 } 3682 return 0; 3683 } 3684 3685 static int 3686 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3687 struct mlxsw_sp_nexthop_group *nh_grp) 3688 { 3689 struct mlxsw_sp_fib_entry *fib_entry; 3690 int err; 3691 3692 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3693 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3694 if (err) 3695 return err; 3696 } 3697 return 0; 3698 } 3699 3700 struct mlxsw_sp_adj_grp_size_range { 3701 u16 start; /* Inclusive */ 3702 u16 end; /* Inclusive */ 3703 }; 3704 3705 /* Ordered by range start value */ 3706 static const struct mlxsw_sp_adj_grp_size_range 3707 mlxsw_sp1_adj_grp_size_ranges[] = { 3708 { .start = 1, .end = 64 }, 3709 { .start = 512, .end = 512 }, 3710 { .start = 1024, .end = 1024 }, 3711 { .start = 2048, .end = 2048 }, 3712 { .start = 4096, .end = 4096 }, 3713 }; 3714 3715 /* Ordered by range start value */ 3716 static const struct mlxsw_sp_adj_grp_size_range 3717 mlxsw_sp2_adj_grp_size_ranges[] = { 3718 { .start = 1, .end = 128 }, 3719 { .start = 256, .end = 256 }, 3720 { .start = 512, .end = 512 }, 3721 { .start = 1024, .end = 1024 }, 3722 { .start = 2048, .end = 2048 }, 3723 { .start = 4096, .end = 4096 }, 3724 }; 3725 3726 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3727 u16 *p_adj_grp_size) 3728 { 3729 int i; 3730 3731 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3732 const struct mlxsw_sp_adj_grp_size_range *size_range; 3733 3734 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3735 3736 if (*p_adj_grp_size >= size_range->start && 3737 *p_adj_grp_size <= size_range->end) 3738 return; 3739 3740 if (*p_adj_grp_size <= size_range->end) { 3741 *p_adj_grp_size = size_range->end; 3742 return; 3743 } 3744 } 3745 } 3746 3747 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3748 u16 *p_adj_grp_size, 3749 unsigned int alloc_size) 3750 { 3751 int i; 3752 3753 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3754 const struct mlxsw_sp_adj_grp_size_range *size_range; 3755 3756 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3757 3758 if (alloc_size >= size_range->end) { 3759 *p_adj_grp_size = size_range->end; 3760 return; 3761 } 3762 } 3763 } 3764 3765 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3766 u16 *p_adj_grp_size) 3767 { 3768 unsigned int alloc_size; 3769 int err; 3770 3771 /* Round up the requested group size to the next size supported 3772 * by the device and make sure the request can be satisfied. 3773 */ 3774 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3775 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3776 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3777 *p_adj_grp_size, &alloc_size); 3778 if (err) 3779 return err; 3780 /* It is possible the allocation results in more allocated 3781 * entries than requested. Try to use as much of them as 3782 * possible. 3783 */ 3784 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3785 3786 return 0; 3787 } 3788 3789 static void 3790 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3791 { 3792 int i, g = 0, sum_norm_weight = 0; 3793 struct mlxsw_sp_nexthop *nh; 3794 3795 for (i = 0; i < nhgi->count; i++) { 3796 nh = &nhgi->nexthops[i]; 3797 3798 if (!nh->should_offload) 3799 continue; 3800 if (g > 0) 3801 g = gcd(nh->nh_weight, g); 3802 else 3803 g = nh->nh_weight; 3804 } 3805 3806 for (i = 0; i < nhgi->count; i++) { 3807 nh = &nhgi->nexthops[i]; 3808 3809 if (!nh->should_offload) 3810 continue; 3811 nh->norm_nh_weight = nh->nh_weight / g; 3812 sum_norm_weight += nh->norm_nh_weight; 3813 } 3814 3815 nhgi->sum_norm_weight = sum_norm_weight; 3816 } 3817 3818 static void 3819 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3820 { 3821 int i, weight = 0, lower_bound = 0; 3822 int total = nhgi->sum_norm_weight; 3823 u16 ecmp_size = nhgi->ecmp_size; 3824 3825 for (i = 0; i < nhgi->count; i++) { 3826 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3827 int upper_bound; 3828 3829 if (!nh->should_offload) 3830 continue; 3831 weight += nh->norm_nh_weight; 3832 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3833 nh->num_adj_entries = upper_bound - lower_bound; 3834 lower_bound = upper_bound; 3835 } 3836 } 3837 3838 static struct mlxsw_sp_nexthop * 3839 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3840 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3841 3842 static void 3843 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3844 struct mlxsw_sp_nexthop_group *nh_grp) 3845 { 3846 int i; 3847 3848 for (i = 0; i < nh_grp->nhgi->count; i++) { 3849 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3850 3851 if (nh->offloaded) 3852 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3853 else 3854 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3855 } 3856 } 3857 3858 static void 3859 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3860 struct mlxsw_sp_fib6_entry *fib6_entry) 3861 { 3862 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3863 3864 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3865 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3866 struct mlxsw_sp_nexthop *nh; 3867 3868 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3869 if (nh && nh->offloaded) 3870 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3871 else 3872 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3873 } 3874 } 3875 3876 static void 3877 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3878 struct mlxsw_sp_nexthop_group *nh_grp) 3879 { 3880 struct mlxsw_sp_fib6_entry *fib6_entry; 3881 3882 /* Unfortunately, in IPv6 the route and the nexthop are described by 3883 * the same struct, so we need to iterate over all the routes using the 3884 * nexthop group and set / clear the offload indication for them. 3885 */ 3886 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3887 common.nexthop_group_node) 3888 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3889 } 3890 3891 static void 3892 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3893 const struct mlxsw_sp_nexthop *nh, 3894 u16 bucket_index) 3895 { 3896 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3897 bool offload = false, trap = false; 3898 3899 if (nh->offloaded) { 3900 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3901 trap = true; 3902 else 3903 offload = true; 3904 } 3905 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3906 bucket_index, offload, trap); 3907 } 3908 3909 static void 3910 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3911 struct mlxsw_sp_nexthop_group *nh_grp) 3912 { 3913 int i; 3914 3915 /* Do not update the flags if the nexthop group is being destroyed 3916 * since: 3917 * 1. The nexthop objects is being deleted, in which case the flags are 3918 * irrelevant. 3919 * 2. The nexthop group was replaced by a newer group, in which case 3920 * the flags of the nexthop object were already updated based on the 3921 * new group. 3922 */ 3923 if (nh_grp->can_destroy) 3924 return; 3925 3926 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3927 nh_grp->nhgi->adj_index_valid, false); 3928 3929 /* Update flags of individual nexthop buckets in case of a resilient 3930 * nexthop group. 3931 */ 3932 if (!nh_grp->nhgi->is_resilient) 3933 return; 3934 3935 for (i = 0; i < nh_grp->nhgi->count; i++) { 3936 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3937 3938 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3939 } 3940 } 3941 3942 static void 3943 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3944 struct mlxsw_sp_nexthop_group *nh_grp) 3945 { 3946 switch (nh_grp->type) { 3947 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3948 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3949 break; 3950 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3951 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3952 break; 3953 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3954 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3955 break; 3956 } 3957 } 3958 3959 static int 3960 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3961 struct mlxsw_sp_nexthop_group *nh_grp) 3962 { 3963 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3964 u16 ecmp_size, old_ecmp_size; 3965 struct mlxsw_sp_nexthop *nh; 3966 bool offload_change = false; 3967 u32 adj_index; 3968 bool old_adj_index_valid; 3969 u32 old_adj_index; 3970 int i, err2, err; 3971 3972 if (!nhgi->gateway) 3973 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3974 3975 for (i = 0; i < nhgi->count; i++) { 3976 nh = &nhgi->nexthops[i]; 3977 3978 if (nh->should_offload != nh->offloaded) { 3979 offload_change = true; 3980 if (nh->should_offload) 3981 nh->update = 1; 3982 } 3983 } 3984 if (!offload_change) { 3985 /* Nothing was added or removed, so no need to reallocate. Just 3986 * update MAC on existing adjacency indexes. 3987 */ 3988 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3989 if (err) { 3990 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3991 goto set_trap; 3992 } 3993 /* Flags of individual nexthop buckets might need to be 3994 * updated. 3995 */ 3996 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3997 return 0; 3998 } 3999 mlxsw_sp_nexthop_group_normalize(nhgi); 4000 if (!nhgi->sum_norm_weight) { 4001 /* No neigh of this group is connected so we just set 4002 * the trap and let everthing flow through kernel. 4003 */ 4004 err = 0; 4005 goto set_trap; 4006 } 4007 4008 ecmp_size = nhgi->sum_norm_weight; 4009 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4010 if (err) 4011 /* No valid allocation size available. */ 4012 goto set_trap; 4013 4014 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4015 ecmp_size, &adj_index); 4016 if (err) { 4017 /* We ran out of KVD linear space, just set the 4018 * trap and let everything flow through kernel. 4019 */ 4020 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4021 goto set_trap; 4022 } 4023 old_adj_index_valid = nhgi->adj_index_valid; 4024 old_adj_index = nhgi->adj_index; 4025 old_ecmp_size = nhgi->ecmp_size; 4026 nhgi->adj_index_valid = 1; 4027 nhgi->adj_index = adj_index; 4028 nhgi->ecmp_size = ecmp_size; 4029 mlxsw_sp_nexthop_group_rebalance(nhgi); 4030 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4031 if (err) { 4032 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4033 goto set_trap; 4034 } 4035 4036 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4037 4038 if (!old_adj_index_valid) { 4039 /* The trap was set for fib entries, so we have to call 4040 * fib entry update to unset it and use adjacency index. 4041 */ 4042 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4043 if (err) { 4044 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4045 goto set_trap; 4046 } 4047 return 0; 4048 } 4049 4050 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4051 old_adj_index, old_ecmp_size); 4052 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4053 old_ecmp_size, old_adj_index); 4054 if (err) { 4055 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4056 goto set_trap; 4057 } 4058 4059 return 0; 4060 4061 set_trap: 4062 old_adj_index_valid = nhgi->adj_index_valid; 4063 nhgi->adj_index_valid = 0; 4064 for (i = 0; i < nhgi->count; i++) { 4065 nh = &nhgi->nexthops[i]; 4066 nh->offloaded = 0; 4067 } 4068 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4069 if (err2) 4070 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4071 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4072 if (old_adj_index_valid) 4073 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4074 nhgi->ecmp_size, nhgi->adj_index); 4075 return err; 4076 } 4077 4078 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4079 bool removing) 4080 { 4081 if (!removing) { 4082 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4083 nh->should_offload = 1; 4084 } else if (nh->nhgi->is_resilient) { 4085 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4086 nh->should_offload = 1; 4087 } else { 4088 nh->should_offload = 0; 4089 } 4090 nh->update = 1; 4091 } 4092 4093 static int 4094 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4095 struct mlxsw_sp_neigh_entry *neigh_entry) 4096 { 4097 struct neighbour *n, *old_n = neigh_entry->key.n; 4098 struct mlxsw_sp_nexthop *nh; 4099 bool entry_connected; 4100 u8 nud_state, dead; 4101 int err; 4102 4103 nh = list_first_entry(&neigh_entry->nexthop_list, 4104 struct mlxsw_sp_nexthop, neigh_list_node); 4105 4106 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4107 if (!n) { 4108 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4109 if (IS_ERR(n)) 4110 return PTR_ERR(n); 4111 neigh_event_send(n, NULL); 4112 } 4113 4114 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4115 neigh_entry->key.n = n; 4116 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4117 if (err) 4118 goto err_neigh_entry_insert; 4119 4120 read_lock_bh(&n->lock); 4121 nud_state = n->nud_state; 4122 dead = n->dead; 4123 read_unlock_bh(&n->lock); 4124 entry_connected = nud_state & NUD_VALID && !dead; 4125 4126 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4127 neigh_list_node) { 4128 neigh_release(old_n); 4129 neigh_clone(n); 4130 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4131 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4132 } 4133 4134 neigh_release(n); 4135 4136 return 0; 4137 4138 err_neigh_entry_insert: 4139 neigh_entry->key.n = old_n; 4140 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4141 neigh_release(n); 4142 return err; 4143 } 4144 4145 static void 4146 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4147 struct mlxsw_sp_neigh_entry *neigh_entry, 4148 bool removing, bool dead) 4149 { 4150 struct mlxsw_sp_nexthop *nh; 4151 4152 if (list_empty(&neigh_entry->nexthop_list)) 4153 return; 4154 4155 if (dead) { 4156 int err; 4157 4158 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4159 neigh_entry); 4160 if (err) 4161 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4162 return; 4163 } 4164 4165 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4166 neigh_list_node) { 4167 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4168 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4169 } 4170 } 4171 4172 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 4173 struct mlxsw_sp_rif *rif) 4174 { 4175 if (nh->rif) 4176 return; 4177 4178 nh->rif = rif; 4179 list_add(&nh->rif_list_node, &rif->nexthop_list); 4180 } 4181 4182 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4183 { 4184 if (!nh->rif) 4185 return; 4186 4187 list_del(&nh->rif_list_node); 4188 nh->rif = NULL; 4189 } 4190 4191 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4192 struct mlxsw_sp_nexthop *nh) 4193 { 4194 struct mlxsw_sp_neigh_entry *neigh_entry; 4195 struct neighbour *n; 4196 u8 nud_state, dead; 4197 int err; 4198 4199 if (!nh->nhgi->gateway || nh->neigh_entry) 4200 return 0; 4201 4202 /* Take a reference of neigh here ensuring that neigh would 4203 * not be destructed before the nexthop entry is finished. 4204 * The reference is taken either in neigh_lookup() or 4205 * in neigh_create() in case n is not found. 4206 */ 4207 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4208 if (!n) { 4209 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4210 if (IS_ERR(n)) 4211 return PTR_ERR(n); 4212 neigh_event_send(n, NULL); 4213 } 4214 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4215 if (!neigh_entry) { 4216 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4217 if (IS_ERR(neigh_entry)) { 4218 err = -EINVAL; 4219 goto err_neigh_entry_create; 4220 } 4221 } 4222 4223 /* If that is the first nexthop connected to that neigh, add to 4224 * nexthop_neighs_list 4225 */ 4226 if (list_empty(&neigh_entry->nexthop_list)) 4227 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4228 &mlxsw_sp->router->nexthop_neighs_list); 4229 4230 nh->neigh_entry = neigh_entry; 4231 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4232 read_lock_bh(&n->lock); 4233 nud_state = n->nud_state; 4234 dead = n->dead; 4235 read_unlock_bh(&n->lock); 4236 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4237 4238 return 0; 4239 4240 err_neigh_entry_create: 4241 neigh_release(n); 4242 return err; 4243 } 4244 4245 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4246 struct mlxsw_sp_nexthop *nh) 4247 { 4248 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4249 struct neighbour *n; 4250 4251 if (!neigh_entry) 4252 return; 4253 n = neigh_entry->key.n; 4254 4255 __mlxsw_sp_nexthop_neigh_update(nh, true); 4256 list_del(&nh->neigh_list_node); 4257 nh->neigh_entry = NULL; 4258 4259 /* If that is the last nexthop connected to that neigh, remove from 4260 * nexthop_neighs_list 4261 */ 4262 if (list_empty(&neigh_entry->nexthop_list)) 4263 list_del(&neigh_entry->nexthop_neighs_list_node); 4264 4265 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4266 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4267 4268 neigh_release(n); 4269 } 4270 4271 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4272 { 4273 struct net_device *ul_dev; 4274 bool is_up; 4275 4276 rcu_read_lock(); 4277 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4278 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4279 rcu_read_unlock(); 4280 4281 return is_up; 4282 } 4283 4284 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4285 struct mlxsw_sp_nexthop *nh, 4286 struct mlxsw_sp_ipip_entry *ipip_entry) 4287 { 4288 bool removing; 4289 4290 if (!nh->nhgi->gateway || nh->ipip_entry) 4291 return; 4292 4293 nh->ipip_entry = ipip_entry; 4294 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4295 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4296 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4297 } 4298 4299 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4300 struct mlxsw_sp_nexthop *nh) 4301 { 4302 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4303 4304 if (!ipip_entry) 4305 return; 4306 4307 __mlxsw_sp_nexthop_neigh_update(nh, true); 4308 nh->ipip_entry = NULL; 4309 } 4310 4311 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4312 const struct fib_nh *fib_nh, 4313 enum mlxsw_sp_ipip_type *p_ipipt) 4314 { 4315 struct net_device *dev = fib_nh->fib_nh_dev; 4316 4317 return dev && 4318 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4319 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4320 } 4321 4322 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4323 struct mlxsw_sp_nexthop *nh, 4324 const struct net_device *dev) 4325 { 4326 const struct mlxsw_sp_ipip_ops *ipip_ops; 4327 struct mlxsw_sp_ipip_entry *ipip_entry; 4328 struct mlxsw_sp_rif *rif; 4329 int err; 4330 4331 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4332 if (ipip_entry) { 4333 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4334 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4335 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4336 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4337 return 0; 4338 } 4339 } 4340 4341 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4342 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4343 if (!rif) 4344 return 0; 4345 4346 mlxsw_sp_nexthop_rif_init(nh, rif); 4347 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4348 if (err) 4349 goto err_neigh_init; 4350 4351 return 0; 4352 4353 err_neigh_init: 4354 mlxsw_sp_nexthop_rif_fini(nh); 4355 return err; 4356 } 4357 4358 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4359 struct mlxsw_sp_nexthop *nh) 4360 { 4361 switch (nh->type) { 4362 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4363 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4364 mlxsw_sp_nexthop_rif_fini(nh); 4365 break; 4366 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4367 mlxsw_sp_nexthop_rif_fini(nh); 4368 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4369 break; 4370 } 4371 } 4372 4373 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4374 struct mlxsw_sp_nexthop_group *nh_grp, 4375 struct mlxsw_sp_nexthop *nh, 4376 struct fib_nh *fib_nh) 4377 { 4378 struct net_device *dev = fib_nh->fib_nh_dev; 4379 struct in_device *in_dev; 4380 int err; 4381 4382 nh->nhgi = nh_grp->nhgi; 4383 nh->key.fib_nh = fib_nh; 4384 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4385 nh->nh_weight = fib_nh->fib_nh_weight; 4386 #else 4387 nh->nh_weight = 1; 4388 #endif 4389 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4390 nh->neigh_tbl = &arp_tbl; 4391 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4392 if (err) 4393 return err; 4394 4395 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4396 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4397 4398 if (!dev) 4399 return 0; 4400 nh->ifindex = dev->ifindex; 4401 4402 rcu_read_lock(); 4403 in_dev = __in_dev_get_rcu(dev); 4404 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4405 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4406 rcu_read_unlock(); 4407 return 0; 4408 } 4409 rcu_read_unlock(); 4410 4411 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4412 if (err) 4413 goto err_nexthop_neigh_init; 4414 4415 return 0; 4416 4417 err_nexthop_neigh_init: 4418 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4419 return err; 4420 } 4421 4422 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4423 struct mlxsw_sp_nexthop *nh) 4424 { 4425 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4426 list_del(&nh->router_list_node); 4427 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4428 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4429 } 4430 4431 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4432 unsigned long event, struct fib_nh *fib_nh) 4433 { 4434 struct mlxsw_sp_nexthop_key key; 4435 struct mlxsw_sp_nexthop *nh; 4436 4437 key.fib_nh = fib_nh; 4438 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4439 if (!nh) 4440 return; 4441 4442 switch (event) { 4443 case FIB_EVENT_NH_ADD: 4444 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4445 break; 4446 case FIB_EVENT_NH_DEL: 4447 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4448 break; 4449 } 4450 4451 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4452 } 4453 4454 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4455 struct mlxsw_sp_rif *rif) 4456 { 4457 struct mlxsw_sp_nexthop *nh; 4458 bool removing; 4459 4460 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4461 switch (nh->type) { 4462 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4463 removing = false; 4464 break; 4465 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4466 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4467 break; 4468 default: 4469 WARN_ON(1); 4470 continue; 4471 } 4472 4473 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4474 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4475 } 4476 } 4477 4478 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4479 struct mlxsw_sp_rif *old_rif, 4480 struct mlxsw_sp_rif *new_rif) 4481 { 4482 struct mlxsw_sp_nexthop *nh; 4483 4484 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4485 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4486 nh->rif = new_rif; 4487 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4488 } 4489 4490 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4491 struct mlxsw_sp_rif *rif) 4492 { 4493 struct mlxsw_sp_nexthop *nh, *tmp; 4494 4495 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4496 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4497 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4498 } 4499 } 4500 4501 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4502 { 4503 enum mlxsw_reg_ratr_trap_action trap_action; 4504 char ratr_pl[MLXSW_REG_RATR_LEN]; 4505 int err; 4506 4507 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4508 &mlxsw_sp->router->adj_trap_index); 4509 if (err) 4510 return err; 4511 4512 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4513 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4514 MLXSW_REG_RATR_TYPE_ETHERNET, 4515 mlxsw_sp->router->adj_trap_index, 4516 mlxsw_sp->router->lb_rif_index); 4517 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4518 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4519 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4520 if (err) 4521 goto err_ratr_write; 4522 4523 return 0; 4524 4525 err_ratr_write: 4526 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4527 mlxsw_sp->router->adj_trap_index); 4528 return err; 4529 } 4530 4531 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4532 { 4533 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4534 mlxsw_sp->router->adj_trap_index); 4535 } 4536 4537 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4538 { 4539 int err; 4540 4541 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4542 return 0; 4543 4544 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4545 if (err) 4546 return err; 4547 4548 refcount_set(&mlxsw_sp->router->num_groups, 1); 4549 4550 return 0; 4551 } 4552 4553 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4554 { 4555 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4556 return; 4557 4558 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4559 } 4560 4561 static void 4562 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4563 const struct mlxsw_sp_nexthop_group *nh_grp, 4564 unsigned long *activity) 4565 { 4566 char *ratrad_pl; 4567 int i, err; 4568 4569 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4570 if (!ratrad_pl) 4571 return; 4572 4573 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4574 nh_grp->nhgi->count); 4575 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4576 if (err) 4577 goto out; 4578 4579 for (i = 0; i < nh_grp->nhgi->count; i++) { 4580 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4581 continue; 4582 bitmap_set(activity, i, 1); 4583 } 4584 4585 out: 4586 kfree(ratrad_pl); 4587 } 4588 4589 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4590 4591 static void 4592 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4593 const struct mlxsw_sp_nexthop_group *nh_grp) 4594 { 4595 unsigned long *activity; 4596 4597 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4598 if (!activity) 4599 return; 4600 4601 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4602 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4603 nh_grp->nhgi->count, activity); 4604 4605 bitmap_free(activity); 4606 } 4607 4608 static void 4609 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4610 { 4611 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4612 4613 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4614 msecs_to_jiffies(interval)); 4615 } 4616 4617 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4618 { 4619 struct mlxsw_sp_nexthop_group_info *nhgi; 4620 struct mlxsw_sp_router *router; 4621 bool reschedule = false; 4622 4623 router = container_of(work, struct mlxsw_sp_router, 4624 nh_grp_activity_dw.work); 4625 4626 mutex_lock(&router->lock); 4627 4628 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4629 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4630 reschedule = true; 4631 } 4632 4633 mutex_unlock(&router->lock); 4634 4635 if (!reschedule) 4636 return; 4637 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4638 } 4639 4640 static int 4641 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4642 const struct nh_notifier_single_info *nh, 4643 struct netlink_ext_ack *extack) 4644 { 4645 int err = -EINVAL; 4646 4647 if (nh->is_fdb) 4648 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4649 else if (nh->has_encap) 4650 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4651 else 4652 err = 0; 4653 4654 return err; 4655 } 4656 4657 static int 4658 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4659 const struct nh_notifier_single_info *nh, 4660 struct netlink_ext_ack *extack) 4661 { 4662 int err; 4663 4664 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4665 if (err) 4666 return err; 4667 4668 /* Device only nexthops with an IPIP device are programmed as 4669 * encapsulating adjacency entries. 4670 */ 4671 if (!nh->gw_family && !nh->is_reject && 4672 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4673 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4674 return -EINVAL; 4675 } 4676 4677 return 0; 4678 } 4679 4680 static int 4681 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4682 const struct nh_notifier_grp_info *nh_grp, 4683 struct netlink_ext_ack *extack) 4684 { 4685 int i; 4686 4687 if (nh_grp->is_fdb) { 4688 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4689 return -EINVAL; 4690 } 4691 4692 for (i = 0; i < nh_grp->num_nh; i++) { 4693 const struct nh_notifier_single_info *nh; 4694 int err; 4695 4696 nh = &nh_grp->nh_entries[i].nh; 4697 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4698 extack); 4699 if (err) 4700 return err; 4701 } 4702 4703 return 0; 4704 } 4705 4706 static int 4707 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4708 const struct nh_notifier_res_table_info *nh_res_table, 4709 struct netlink_ext_ack *extack) 4710 { 4711 unsigned int alloc_size; 4712 bool valid_size = false; 4713 int err, i; 4714 4715 if (nh_res_table->num_nh_buckets < 32) { 4716 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4717 return -EINVAL; 4718 } 4719 4720 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4721 const struct mlxsw_sp_adj_grp_size_range *size_range; 4722 4723 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4724 4725 if (nh_res_table->num_nh_buckets >= size_range->start && 4726 nh_res_table->num_nh_buckets <= size_range->end) { 4727 valid_size = true; 4728 break; 4729 } 4730 } 4731 4732 if (!valid_size) { 4733 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4734 return -EINVAL; 4735 } 4736 4737 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4738 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4739 nh_res_table->num_nh_buckets, 4740 &alloc_size); 4741 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4742 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4743 return -EINVAL; 4744 } 4745 4746 return 0; 4747 } 4748 4749 static int 4750 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4751 const struct nh_notifier_res_table_info *nh_res_table, 4752 struct netlink_ext_ack *extack) 4753 { 4754 int err; 4755 u16 i; 4756 4757 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4758 nh_res_table, 4759 extack); 4760 if (err) 4761 return err; 4762 4763 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4764 const struct nh_notifier_single_info *nh; 4765 int err; 4766 4767 nh = &nh_res_table->nhs[i]; 4768 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4769 extack); 4770 if (err) 4771 return err; 4772 } 4773 4774 return 0; 4775 } 4776 4777 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4778 unsigned long event, 4779 struct nh_notifier_info *info) 4780 { 4781 struct nh_notifier_single_info *nh; 4782 4783 if (event != NEXTHOP_EVENT_REPLACE && 4784 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4785 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4786 return 0; 4787 4788 switch (info->type) { 4789 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4790 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4791 info->extack); 4792 case NH_NOTIFIER_INFO_TYPE_GRP: 4793 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4794 info->nh_grp, 4795 info->extack); 4796 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4797 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4798 info->nh_res_table, 4799 info->extack); 4800 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4801 nh = &info->nh_res_bucket->new_nh; 4802 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4803 info->extack); 4804 default: 4805 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4806 return -EOPNOTSUPP; 4807 } 4808 } 4809 4810 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4811 const struct nh_notifier_info *info) 4812 { 4813 const struct net_device *dev; 4814 4815 switch (info->type) { 4816 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4817 dev = info->nh->dev; 4818 return info->nh->gw_family || info->nh->is_reject || 4819 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4820 case NH_NOTIFIER_INFO_TYPE_GRP: 4821 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4822 /* Already validated earlier. */ 4823 return true; 4824 default: 4825 return false; 4826 } 4827 } 4828 4829 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4830 struct mlxsw_sp_nexthop *nh) 4831 { 4832 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4833 4834 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4835 nh->should_offload = 1; 4836 /* While nexthops that discard packets do not forward packets 4837 * via an egress RIF, they still need to be programmed using a 4838 * valid RIF, so use the loopback RIF created during init. 4839 */ 4840 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4841 } 4842 4843 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4844 struct mlxsw_sp_nexthop *nh) 4845 { 4846 nh->rif = NULL; 4847 nh->should_offload = 0; 4848 } 4849 4850 static int 4851 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4852 struct mlxsw_sp_nexthop_group *nh_grp, 4853 struct mlxsw_sp_nexthop *nh, 4854 struct nh_notifier_single_info *nh_obj, int weight) 4855 { 4856 struct net_device *dev = nh_obj->dev; 4857 int err; 4858 4859 nh->nhgi = nh_grp->nhgi; 4860 nh->nh_weight = weight; 4861 4862 switch (nh_obj->gw_family) { 4863 case AF_INET: 4864 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4865 nh->neigh_tbl = &arp_tbl; 4866 break; 4867 case AF_INET6: 4868 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4869 #if IS_ENABLED(CONFIG_IPV6) 4870 nh->neigh_tbl = &nd_tbl; 4871 #endif 4872 break; 4873 } 4874 4875 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4876 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4877 nh->ifindex = dev->ifindex; 4878 4879 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4880 if (err) 4881 goto err_type_init; 4882 4883 if (nh_obj->is_reject) 4884 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4885 4886 /* In a resilient nexthop group, all the nexthops must be written to 4887 * the adjacency table. Even if they do not have a valid neighbour or 4888 * RIF. 4889 */ 4890 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4891 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4892 nh->should_offload = 1; 4893 } 4894 4895 return 0; 4896 4897 err_type_init: 4898 list_del(&nh->router_list_node); 4899 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4900 return err; 4901 } 4902 4903 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4904 struct mlxsw_sp_nexthop *nh) 4905 { 4906 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4907 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4908 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4909 list_del(&nh->router_list_node); 4910 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4911 nh->should_offload = 0; 4912 } 4913 4914 static int 4915 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4916 struct mlxsw_sp_nexthop_group *nh_grp, 4917 struct nh_notifier_info *info) 4918 { 4919 struct mlxsw_sp_nexthop_group_info *nhgi; 4920 struct mlxsw_sp_nexthop *nh; 4921 bool is_resilient = false; 4922 unsigned int nhs; 4923 int err, i; 4924 4925 switch (info->type) { 4926 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4927 nhs = 1; 4928 break; 4929 case NH_NOTIFIER_INFO_TYPE_GRP: 4930 nhs = info->nh_grp->num_nh; 4931 break; 4932 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4933 nhs = info->nh_res_table->num_nh_buckets; 4934 is_resilient = true; 4935 break; 4936 default: 4937 return -EINVAL; 4938 } 4939 4940 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4941 if (!nhgi) 4942 return -ENOMEM; 4943 nh_grp->nhgi = nhgi; 4944 nhgi->nh_grp = nh_grp; 4945 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4946 nhgi->is_resilient = is_resilient; 4947 nhgi->count = nhs; 4948 for (i = 0; i < nhgi->count; i++) { 4949 struct nh_notifier_single_info *nh_obj; 4950 int weight; 4951 4952 nh = &nhgi->nexthops[i]; 4953 switch (info->type) { 4954 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4955 nh_obj = info->nh; 4956 weight = 1; 4957 break; 4958 case NH_NOTIFIER_INFO_TYPE_GRP: 4959 nh_obj = &info->nh_grp->nh_entries[i].nh; 4960 weight = info->nh_grp->nh_entries[i].weight; 4961 break; 4962 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4963 nh_obj = &info->nh_res_table->nhs[i]; 4964 weight = 1; 4965 break; 4966 default: 4967 err = -EINVAL; 4968 goto err_nexthop_obj_init; 4969 } 4970 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4971 weight); 4972 if (err) 4973 goto err_nexthop_obj_init; 4974 } 4975 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 4976 if (err) 4977 goto err_group_inc; 4978 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4979 if (err) { 4980 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4981 goto err_group_refresh; 4982 } 4983 4984 /* Add resilient nexthop groups to a list so that the activity of their 4985 * nexthop buckets will be periodically queried and cleared. 4986 */ 4987 if (nhgi->is_resilient) { 4988 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4989 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 4990 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 4991 } 4992 4993 return 0; 4994 4995 err_group_refresh: 4996 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 4997 err_group_inc: 4998 i = nhgi->count; 4999 err_nexthop_obj_init: 5000 for (i--; i >= 0; i--) { 5001 nh = &nhgi->nexthops[i]; 5002 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5003 } 5004 kfree(nhgi); 5005 return err; 5006 } 5007 5008 static void 5009 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5010 struct mlxsw_sp_nexthop_group *nh_grp) 5011 { 5012 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5013 struct mlxsw_sp_router *router = mlxsw_sp->router; 5014 int i; 5015 5016 if (nhgi->is_resilient) { 5017 list_del(&nhgi->list); 5018 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5019 cancel_delayed_work(&router->nh_grp_activity_dw); 5020 } 5021 5022 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5023 for (i = nhgi->count - 1; i >= 0; i--) { 5024 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5025 5026 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5027 } 5028 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5029 WARN_ON_ONCE(nhgi->adj_index_valid); 5030 kfree(nhgi); 5031 } 5032 5033 static struct mlxsw_sp_nexthop_group * 5034 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5035 struct nh_notifier_info *info) 5036 { 5037 struct mlxsw_sp_nexthop_group *nh_grp; 5038 int err; 5039 5040 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5041 if (!nh_grp) 5042 return ERR_PTR(-ENOMEM); 5043 INIT_LIST_HEAD(&nh_grp->vr_list); 5044 err = rhashtable_init(&nh_grp->vr_ht, 5045 &mlxsw_sp_nexthop_group_vr_ht_params); 5046 if (err) 5047 goto err_nexthop_group_vr_ht_init; 5048 INIT_LIST_HEAD(&nh_grp->fib_list); 5049 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5050 nh_grp->obj.id = info->id; 5051 5052 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5053 if (err) 5054 goto err_nexthop_group_info_init; 5055 5056 nh_grp->can_destroy = false; 5057 5058 return nh_grp; 5059 5060 err_nexthop_group_info_init: 5061 rhashtable_destroy(&nh_grp->vr_ht); 5062 err_nexthop_group_vr_ht_init: 5063 kfree(nh_grp); 5064 return ERR_PTR(err); 5065 } 5066 5067 static void 5068 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5069 struct mlxsw_sp_nexthop_group *nh_grp) 5070 { 5071 if (!nh_grp->can_destroy) 5072 return; 5073 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5074 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5075 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5076 rhashtable_destroy(&nh_grp->vr_ht); 5077 kfree(nh_grp); 5078 } 5079 5080 static struct mlxsw_sp_nexthop_group * 5081 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5082 { 5083 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5084 5085 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5086 cmp_arg.id = id; 5087 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5088 &cmp_arg, 5089 mlxsw_sp_nexthop_group_ht_params); 5090 } 5091 5092 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5093 struct mlxsw_sp_nexthop_group *nh_grp) 5094 { 5095 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5096 } 5097 5098 static int 5099 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5100 struct mlxsw_sp_nexthop_group *nh_grp, 5101 struct mlxsw_sp_nexthop_group *old_nh_grp, 5102 struct netlink_ext_ack *extack) 5103 { 5104 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5105 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5106 int err; 5107 5108 old_nh_grp->nhgi = new_nhgi; 5109 new_nhgi->nh_grp = old_nh_grp; 5110 nh_grp->nhgi = old_nhgi; 5111 old_nhgi->nh_grp = nh_grp; 5112 5113 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5114 /* Both the old adjacency index and the new one are valid. 5115 * Routes are currently using the old one. Tell the device to 5116 * replace the old adjacency index with the new one. 5117 */ 5118 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5119 old_nhgi->adj_index, 5120 old_nhgi->ecmp_size); 5121 if (err) { 5122 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5123 goto err_out; 5124 } 5125 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5126 /* The old adjacency index is valid, while the new one is not. 5127 * Iterate over all the routes using the group and change them 5128 * to trap packets to the CPU. 5129 */ 5130 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5131 if (err) { 5132 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5133 goto err_out; 5134 } 5135 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5136 /* The old adjacency index is invalid, while the new one is. 5137 * Iterate over all the routes using the group and change them 5138 * to forward packets using the new valid index. 5139 */ 5140 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5141 if (err) { 5142 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5143 goto err_out; 5144 } 5145 } 5146 5147 /* Make sure the flags are set / cleared based on the new nexthop group 5148 * information. 5149 */ 5150 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5151 5152 /* At this point 'nh_grp' is just a shell that is not used by anyone 5153 * and its nexthop group info is the old info that was just replaced 5154 * with the new one. Remove it. 5155 */ 5156 nh_grp->can_destroy = true; 5157 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5158 5159 return 0; 5160 5161 err_out: 5162 old_nhgi->nh_grp = old_nh_grp; 5163 nh_grp->nhgi = new_nhgi; 5164 new_nhgi->nh_grp = nh_grp; 5165 old_nh_grp->nhgi = old_nhgi; 5166 return err; 5167 } 5168 5169 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5170 struct nh_notifier_info *info) 5171 { 5172 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5173 struct netlink_ext_ack *extack = info->extack; 5174 int err; 5175 5176 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5177 if (IS_ERR(nh_grp)) 5178 return PTR_ERR(nh_grp); 5179 5180 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5181 if (!old_nh_grp) 5182 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5183 else 5184 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5185 old_nh_grp, extack); 5186 5187 if (err) { 5188 nh_grp->can_destroy = true; 5189 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5190 } 5191 5192 return err; 5193 } 5194 5195 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5196 struct nh_notifier_info *info) 5197 { 5198 struct mlxsw_sp_nexthop_group *nh_grp; 5199 5200 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5201 if (!nh_grp) 5202 return; 5203 5204 nh_grp->can_destroy = true; 5205 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5206 5207 /* If the group still has routes using it, then defer the delete 5208 * operation until the last route using it is deleted. 5209 */ 5210 if (!list_empty(&nh_grp->fib_list)) 5211 return; 5212 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5213 } 5214 5215 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5216 u32 adj_index, char *ratr_pl) 5217 { 5218 MLXSW_REG_ZERO(ratr, ratr_pl); 5219 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5220 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5221 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5222 5223 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5224 } 5225 5226 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5227 { 5228 /* Clear the opcode and activity on both the old and new payload as 5229 * they are irrelevant for the comparison. 5230 */ 5231 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5232 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5233 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5234 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5235 5236 /* If the contents of the adjacency entry are consistent with the 5237 * replacement request, then replacement was successful. 5238 */ 5239 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5240 return 0; 5241 5242 return -EINVAL; 5243 } 5244 5245 static int 5246 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5247 struct mlxsw_sp_nexthop *nh, 5248 struct nh_notifier_info *info) 5249 { 5250 u16 bucket_index = info->nh_res_bucket->bucket_index; 5251 struct netlink_ext_ack *extack = info->extack; 5252 bool force = info->nh_res_bucket->force; 5253 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5254 char ratr_pl[MLXSW_REG_RATR_LEN]; 5255 u32 adj_index; 5256 int err; 5257 5258 /* No point in trying an atomic replacement if the idle timer interval 5259 * is smaller than the interval in which we query and clear activity. 5260 */ 5261 if (!force && info->nh_res_bucket->idle_timer_ms < 5262 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5263 force = true; 5264 5265 adj_index = nh->nhgi->adj_index + bucket_index; 5266 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5267 if (err) { 5268 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5269 return err; 5270 } 5271 5272 if (!force) { 5273 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5274 ratr_pl_new); 5275 if (err) { 5276 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5277 return err; 5278 } 5279 5280 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5281 if (err) { 5282 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5283 return err; 5284 } 5285 } 5286 5287 nh->update = 0; 5288 nh->offloaded = 1; 5289 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5290 5291 return 0; 5292 } 5293 5294 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5295 struct nh_notifier_info *info) 5296 { 5297 u16 bucket_index = info->nh_res_bucket->bucket_index; 5298 struct netlink_ext_ack *extack = info->extack; 5299 struct mlxsw_sp_nexthop_group_info *nhgi; 5300 struct nh_notifier_single_info *nh_obj; 5301 struct mlxsw_sp_nexthop_group *nh_grp; 5302 struct mlxsw_sp_nexthop *nh; 5303 int err; 5304 5305 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5306 if (!nh_grp) { 5307 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5308 return -EINVAL; 5309 } 5310 5311 nhgi = nh_grp->nhgi; 5312 5313 if (bucket_index >= nhgi->count) { 5314 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5315 return -EINVAL; 5316 } 5317 5318 nh = &nhgi->nexthops[bucket_index]; 5319 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5320 5321 nh_obj = &info->nh_res_bucket->new_nh; 5322 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5323 if (err) { 5324 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5325 goto err_nexthop_obj_init; 5326 } 5327 5328 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5329 if (err) 5330 goto err_nexthop_obj_bucket_adj_update; 5331 5332 return 0; 5333 5334 err_nexthop_obj_bucket_adj_update: 5335 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5336 err_nexthop_obj_init: 5337 nh_obj = &info->nh_res_bucket->old_nh; 5338 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5339 /* The old adjacency entry was not overwritten */ 5340 nh->update = 0; 5341 nh->offloaded = 1; 5342 return err; 5343 } 5344 5345 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5346 unsigned long event, void *ptr) 5347 { 5348 struct nh_notifier_info *info = ptr; 5349 struct mlxsw_sp_router *router; 5350 int err = 0; 5351 5352 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5353 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5354 if (err) 5355 goto out; 5356 5357 mutex_lock(&router->lock); 5358 5359 switch (event) { 5360 case NEXTHOP_EVENT_REPLACE: 5361 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5362 break; 5363 case NEXTHOP_EVENT_DEL: 5364 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5365 break; 5366 case NEXTHOP_EVENT_BUCKET_REPLACE: 5367 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5368 info); 5369 break; 5370 default: 5371 break; 5372 } 5373 5374 mutex_unlock(&router->lock); 5375 5376 out: 5377 return notifier_from_errno(err); 5378 } 5379 5380 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5381 struct fib_info *fi) 5382 { 5383 const struct fib_nh *nh = fib_info_nh(fi, 0); 5384 5385 return nh->fib_nh_scope == RT_SCOPE_LINK || 5386 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5387 } 5388 5389 static int 5390 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5391 struct mlxsw_sp_nexthop_group *nh_grp) 5392 { 5393 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5394 struct mlxsw_sp_nexthop_group_info *nhgi; 5395 struct mlxsw_sp_nexthop *nh; 5396 int err, i; 5397 5398 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5399 if (!nhgi) 5400 return -ENOMEM; 5401 nh_grp->nhgi = nhgi; 5402 nhgi->nh_grp = nh_grp; 5403 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5404 nhgi->count = nhs; 5405 for (i = 0; i < nhgi->count; i++) { 5406 struct fib_nh *fib_nh; 5407 5408 nh = &nhgi->nexthops[i]; 5409 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5410 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5411 if (err) 5412 goto err_nexthop4_init; 5413 } 5414 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5415 if (err) 5416 goto err_group_inc; 5417 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5418 if (err) 5419 goto err_group_refresh; 5420 5421 return 0; 5422 5423 err_group_refresh: 5424 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5425 err_group_inc: 5426 i = nhgi->count; 5427 err_nexthop4_init: 5428 for (i--; i >= 0; i--) { 5429 nh = &nhgi->nexthops[i]; 5430 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5431 } 5432 kfree(nhgi); 5433 return err; 5434 } 5435 5436 static void 5437 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5438 struct mlxsw_sp_nexthop_group *nh_grp) 5439 { 5440 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5441 int i; 5442 5443 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5444 for (i = nhgi->count - 1; i >= 0; i--) { 5445 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5446 5447 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5448 } 5449 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5450 WARN_ON_ONCE(nhgi->adj_index_valid); 5451 kfree(nhgi); 5452 } 5453 5454 static struct mlxsw_sp_nexthop_group * 5455 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5456 { 5457 struct mlxsw_sp_nexthop_group *nh_grp; 5458 int err; 5459 5460 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5461 if (!nh_grp) 5462 return ERR_PTR(-ENOMEM); 5463 INIT_LIST_HEAD(&nh_grp->vr_list); 5464 err = rhashtable_init(&nh_grp->vr_ht, 5465 &mlxsw_sp_nexthop_group_vr_ht_params); 5466 if (err) 5467 goto err_nexthop_group_vr_ht_init; 5468 INIT_LIST_HEAD(&nh_grp->fib_list); 5469 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5470 nh_grp->ipv4.fi = fi; 5471 fib_info_hold(fi); 5472 5473 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5474 if (err) 5475 goto err_nexthop_group_info_init; 5476 5477 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5478 if (err) 5479 goto err_nexthop_group_insert; 5480 5481 nh_grp->can_destroy = true; 5482 5483 return nh_grp; 5484 5485 err_nexthop_group_insert: 5486 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5487 err_nexthop_group_info_init: 5488 fib_info_put(fi); 5489 rhashtable_destroy(&nh_grp->vr_ht); 5490 err_nexthop_group_vr_ht_init: 5491 kfree(nh_grp); 5492 return ERR_PTR(err); 5493 } 5494 5495 static void 5496 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5497 struct mlxsw_sp_nexthop_group *nh_grp) 5498 { 5499 if (!nh_grp->can_destroy) 5500 return; 5501 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5502 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5503 fib_info_put(nh_grp->ipv4.fi); 5504 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5505 rhashtable_destroy(&nh_grp->vr_ht); 5506 kfree(nh_grp); 5507 } 5508 5509 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5510 struct mlxsw_sp_fib_entry *fib_entry, 5511 struct fib_info *fi) 5512 { 5513 struct mlxsw_sp_nexthop_group *nh_grp; 5514 5515 if (fi->nh) { 5516 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5517 fi->nh->id); 5518 if (WARN_ON_ONCE(!nh_grp)) 5519 return -EINVAL; 5520 goto out; 5521 } 5522 5523 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5524 if (!nh_grp) { 5525 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5526 if (IS_ERR(nh_grp)) 5527 return PTR_ERR(nh_grp); 5528 } 5529 out: 5530 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5531 fib_entry->nh_group = nh_grp; 5532 return 0; 5533 } 5534 5535 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5536 struct mlxsw_sp_fib_entry *fib_entry) 5537 { 5538 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5539 5540 list_del(&fib_entry->nexthop_group_node); 5541 if (!list_empty(&nh_grp->fib_list)) 5542 return; 5543 5544 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5545 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5546 return; 5547 } 5548 5549 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5550 } 5551 5552 static bool 5553 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5554 { 5555 struct mlxsw_sp_fib4_entry *fib4_entry; 5556 5557 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5558 common); 5559 return !fib4_entry->dscp; 5560 } 5561 5562 static bool 5563 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5564 { 5565 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5566 5567 switch (fib_entry->fib_node->fib->proto) { 5568 case MLXSW_SP_L3_PROTO_IPV4: 5569 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5570 return false; 5571 break; 5572 case MLXSW_SP_L3_PROTO_IPV6: 5573 break; 5574 } 5575 5576 switch (fib_entry->type) { 5577 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5578 return !!nh_group->nhgi->adj_index_valid; 5579 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5580 return !!nh_group->nhgi->nh_rif; 5581 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5582 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5583 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5584 return true; 5585 default: 5586 return false; 5587 } 5588 } 5589 5590 static struct mlxsw_sp_nexthop * 5591 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5592 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5593 { 5594 int i; 5595 5596 for (i = 0; i < nh_grp->nhgi->count; i++) { 5597 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5598 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5599 5600 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5601 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5602 &rt->fib6_nh->fib_nh_gw6)) 5603 return nh; 5604 } 5605 5606 return NULL; 5607 } 5608 5609 static void 5610 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5611 struct fib_entry_notifier_info *fen_info) 5612 { 5613 u32 *p_dst = (u32 *) &fen_info->dst; 5614 struct fib_rt_info fri; 5615 5616 fri.fi = fen_info->fi; 5617 fri.tb_id = fen_info->tb_id; 5618 fri.dst = cpu_to_be32(*p_dst); 5619 fri.dst_len = fen_info->dst_len; 5620 fri.dscp = fen_info->dscp; 5621 fri.type = fen_info->type; 5622 fri.offload = false; 5623 fri.trap = false; 5624 fri.offload_failed = true; 5625 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5626 } 5627 5628 static void 5629 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5630 struct mlxsw_sp_fib_entry *fib_entry) 5631 { 5632 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5633 int dst_len = fib_entry->fib_node->key.prefix_len; 5634 struct mlxsw_sp_fib4_entry *fib4_entry; 5635 struct fib_rt_info fri; 5636 bool should_offload; 5637 5638 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5639 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5640 common); 5641 fri.fi = fib4_entry->fi; 5642 fri.tb_id = fib4_entry->tb_id; 5643 fri.dst = cpu_to_be32(*p_dst); 5644 fri.dst_len = dst_len; 5645 fri.dscp = fib4_entry->dscp; 5646 fri.type = fib4_entry->type; 5647 fri.offload = should_offload; 5648 fri.trap = !should_offload; 5649 fri.offload_failed = false; 5650 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5651 } 5652 5653 static void 5654 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5655 struct mlxsw_sp_fib_entry *fib_entry) 5656 { 5657 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5658 int dst_len = fib_entry->fib_node->key.prefix_len; 5659 struct mlxsw_sp_fib4_entry *fib4_entry; 5660 struct fib_rt_info fri; 5661 5662 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5663 common); 5664 fri.fi = fib4_entry->fi; 5665 fri.tb_id = fib4_entry->tb_id; 5666 fri.dst = cpu_to_be32(*p_dst); 5667 fri.dst_len = dst_len; 5668 fri.dscp = fib4_entry->dscp; 5669 fri.type = fib4_entry->type; 5670 fri.offload = false; 5671 fri.trap = false; 5672 fri.offload_failed = false; 5673 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5674 } 5675 5676 #if IS_ENABLED(CONFIG_IPV6) 5677 static void 5678 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5679 struct fib6_info **rt_arr, 5680 unsigned int nrt6) 5681 { 5682 int i; 5683 5684 /* In IPv6 a multipath route is represented using multiple routes, so 5685 * we need to set the flags on all of them. 5686 */ 5687 for (i = 0; i < nrt6; i++) 5688 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5689 false, false, true); 5690 } 5691 #else 5692 static void 5693 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5694 struct fib6_info **rt_arr, 5695 unsigned int nrt6) 5696 { 5697 } 5698 #endif 5699 5700 #if IS_ENABLED(CONFIG_IPV6) 5701 static void 5702 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5703 struct mlxsw_sp_fib_entry *fib_entry) 5704 { 5705 struct mlxsw_sp_fib6_entry *fib6_entry; 5706 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5707 bool should_offload; 5708 5709 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5710 5711 /* In IPv6 a multipath route is represented using multiple routes, so 5712 * we need to set the flags on all of them. 5713 */ 5714 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5715 common); 5716 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5717 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5718 should_offload, !should_offload, false); 5719 } 5720 #else 5721 static void 5722 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5723 struct mlxsw_sp_fib_entry *fib_entry) 5724 { 5725 } 5726 #endif 5727 5728 #if IS_ENABLED(CONFIG_IPV6) 5729 static void 5730 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5731 struct mlxsw_sp_fib_entry *fib_entry) 5732 { 5733 struct mlxsw_sp_fib6_entry *fib6_entry; 5734 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5735 5736 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5737 common); 5738 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5739 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5740 false, false, false); 5741 } 5742 #else 5743 static void 5744 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5745 struct mlxsw_sp_fib_entry *fib_entry) 5746 { 5747 } 5748 #endif 5749 5750 static void 5751 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5752 struct mlxsw_sp_fib_entry *fib_entry) 5753 { 5754 switch (fib_entry->fib_node->fib->proto) { 5755 case MLXSW_SP_L3_PROTO_IPV4: 5756 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5757 break; 5758 case MLXSW_SP_L3_PROTO_IPV6: 5759 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5760 break; 5761 } 5762 } 5763 5764 static void 5765 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5766 struct mlxsw_sp_fib_entry *fib_entry) 5767 { 5768 switch (fib_entry->fib_node->fib->proto) { 5769 case MLXSW_SP_L3_PROTO_IPV4: 5770 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5771 break; 5772 case MLXSW_SP_L3_PROTO_IPV6: 5773 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5774 break; 5775 } 5776 } 5777 5778 static void 5779 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5780 struct mlxsw_sp_fib_entry *fib_entry, 5781 enum mlxsw_sp_fib_entry_op op) 5782 { 5783 switch (op) { 5784 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5785 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5786 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5787 break; 5788 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5789 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5790 break; 5791 default: 5792 break; 5793 } 5794 } 5795 5796 struct mlxsw_sp_fib_entry_op_ctx_basic { 5797 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5798 }; 5799 5800 static void 5801 mlxsw_sp_router_ll_basic_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5802 enum mlxsw_sp_l3proto proto, 5803 enum mlxsw_sp_fib_entry_op op, 5804 u16 virtual_router, u8 prefix_len, 5805 unsigned char *addr, 5806 struct mlxsw_sp_fib_entry_priv *priv) 5807 { 5808 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5809 enum mlxsw_reg_ralxx_protocol ralxx_proto; 5810 char *ralue_pl = op_ctx_basic->ralue_pl; 5811 enum mlxsw_reg_ralue_op ralue_op; 5812 5813 ralxx_proto = (enum mlxsw_reg_ralxx_protocol) proto; 5814 5815 switch (op) { 5816 case MLXSW_SP_FIB_ENTRY_OP_WRITE: 5817 case MLXSW_SP_FIB_ENTRY_OP_UPDATE: 5818 ralue_op = MLXSW_REG_RALUE_OP_WRITE_WRITE; 5819 break; 5820 case MLXSW_SP_FIB_ENTRY_OP_DELETE: 5821 ralue_op = MLXSW_REG_RALUE_OP_WRITE_DELETE; 5822 break; 5823 default: 5824 WARN_ON_ONCE(1); 5825 return; 5826 } 5827 5828 switch (proto) { 5829 case MLXSW_SP_L3_PROTO_IPV4: 5830 mlxsw_reg_ralue_pack4(ralue_pl, ralxx_proto, ralue_op, 5831 virtual_router, prefix_len, (u32 *) addr); 5832 break; 5833 case MLXSW_SP_L3_PROTO_IPV6: 5834 mlxsw_reg_ralue_pack6(ralue_pl, ralxx_proto, ralue_op, 5835 virtual_router, prefix_len, addr); 5836 break; 5837 } 5838 } 5839 5840 static void 5841 mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5842 enum mlxsw_reg_ralue_trap_action trap_action, 5843 u16 trap_id, u32 adjacency_index, u16 ecmp_size) 5844 { 5845 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5846 5847 mlxsw_reg_ralue_act_remote_pack(op_ctx_basic->ralue_pl, trap_action, 5848 trap_id, adjacency_index, ecmp_size); 5849 } 5850 5851 static void 5852 mlxsw_sp_router_ll_basic_fib_entry_act_local_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5853 enum mlxsw_reg_ralue_trap_action trap_action, 5854 u16 trap_id, u16 local_erif) 5855 { 5856 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5857 5858 mlxsw_reg_ralue_act_local_pack(op_ctx_basic->ralue_pl, trap_action, 5859 trap_id, local_erif); 5860 } 5861 5862 static void 5863 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx) 5864 { 5865 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5866 5867 mlxsw_reg_ralue_act_ip2me_pack(op_ctx_basic->ralue_pl); 5868 } 5869 5870 static void 5871 mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5872 u32 tunnel_ptr) 5873 { 5874 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5875 5876 mlxsw_reg_ralue_act_ip2me_tun_pack(op_ctx_basic->ralue_pl, tunnel_ptr); 5877 } 5878 5879 static int 5880 mlxsw_sp_router_ll_basic_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5881 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5882 bool *postponed_for_bulk) 5883 { 5884 struct mlxsw_sp_fib_entry_op_ctx_basic *op_ctx_basic = (void *) op_ctx->ll_priv; 5885 5886 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), 5887 op_ctx_basic->ralue_pl); 5888 } 5889 5890 static bool 5891 mlxsw_sp_router_ll_basic_fib_entry_is_committed(struct mlxsw_sp_fib_entry_priv *priv) 5892 { 5893 return true; 5894 } 5895 5896 static void mlxsw_sp_fib_entry_pack(struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5897 struct mlxsw_sp_fib_entry *fib_entry, 5898 enum mlxsw_sp_fib_entry_op op) 5899 { 5900 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5901 5902 mlxsw_sp_fib_entry_op_ctx_priv_hold(op_ctx, fib_entry->priv); 5903 fib->ll_ops->fib_entry_pack(op_ctx, fib->proto, op, fib->vr->id, 5904 fib_entry->fib_node->key.prefix_len, 5905 fib_entry->fib_node->key.addr, 5906 fib_entry->priv); 5907 } 5908 5909 static int mlxsw_sp_fib_entry_commit(struct mlxsw_sp *mlxsw_sp, 5910 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5911 const struct mlxsw_sp_router_ll_ops *ll_ops) 5912 { 5913 bool postponed_for_bulk = false; 5914 int err; 5915 5916 err = ll_ops->fib_entry_commit(mlxsw_sp, op_ctx, &postponed_for_bulk); 5917 if (!postponed_for_bulk) 5918 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 5919 return err; 5920 } 5921 5922 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5923 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5924 struct mlxsw_sp_fib_entry *fib_entry, 5925 enum mlxsw_sp_fib_entry_op op) 5926 { 5927 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5928 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5929 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5930 enum mlxsw_reg_ralue_trap_action trap_action; 5931 u16 trap_id = 0; 5932 u32 adjacency_index = 0; 5933 u16 ecmp_size = 0; 5934 5935 /* In case the nexthop group adjacency index is valid, use it 5936 * with provided ECMP size. Otherwise, setup trap and pass 5937 * traffic to kernel. 5938 */ 5939 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5940 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5941 adjacency_index = nhgi->adj_index; 5942 ecmp_size = nhgi->ecmp_size; 5943 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5944 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5945 adjacency_index = mlxsw_sp->router->adj_trap_index; 5946 ecmp_size = 1; 5947 } else { 5948 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5949 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5950 } 5951 5952 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5953 ll_ops->fib_entry_act_remote_pack(op_ctx, trap_action, trap_id, 5954 adjacency_index, ecmp_size); 5955 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5956 } 5957 5958 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5959 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5960 struct mlxsw_sp_fib_entry *fib_entry, 5961 enum mlxsw_sp_fib_entry_op op) 5962 { 5963 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5964 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5965 enum mlxsw_reg_ralue_trap_action trap_action; 5966 u16 trap_id = 0; 5967 u16 rif_index = 0; 5968 5969 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5970 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5971 rif_index = rif->rif_index; 5972 } else { 5973 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5974 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5975 } 5976 5977 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5978 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, rif_index); 5979 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5980 } 5981 5982 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5983 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5984 struct mlxsw_sp_fib_entry *fib_entry, 5985 enum mlxsw_sp_fib_entry_op op) 5986 { 5987 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 5988 5989 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 5990 ll_ops->fib_entry_act_ip2me_pack(op_ctx); 5991 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 5992 } 5993 5994 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5995 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 5996 struct mlxsw_sp_fib_entry *fib_entry, 5997 enum mlxsw_sp_fib_entry_op op) 5998 { 5999 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6000 enum mlxsw_reg_ralue_trap_action trap_action; 6001 6002 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 6003 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6004 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, 0, 0); 6005 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6006 } 6007 6008 static int 6009 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 6010 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6011 struct mlxsw_sp_fib_entry *fib_entry, 6012 enum mlxsw_sp_fib_entry_op op) 6013 { 6014 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6015 enum mlxsw_reg_ralue_trap_action trap_action; 6016 u16 trap_id; 6017 6018 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 6019 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 6020 6021 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6022 ll_ops->fib_entry_act_local_pack(op_ctx, trap_action, trap_id, 0); 6023 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6024 } 6025 6026 static int 6027 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 6028 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6029 struct mlxsw_sp_fib_entry *fib_entry, 6030 enum mlxsw_sp_fib_entry_op op) 6031 { 6032 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6033 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 6034 const struct mlxsw_sp_ipip_ops *ipip_ops; 6035 int err; 6036 6037 if (WARN_ON(!ipip_entry)) 6038 return -EINVAL; 6039 6040 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 6041 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 6042 fib_entry->decap.tunnel_index); 6043 if (err) 6044 return err; 6045 6046 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6047 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 6048 fib_entry->decap.tunnel_index); 6049 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6050 } 6051 6052 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 6053 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6054 struct mlxsw_sp_fib_entry *fib_entry, 6055 enum mlxsw_sp_fib_entry_op op) 6056 { 6057 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6058 6059 mlxsw_sp_fib_entry_pack(op_ctx, fib_entry, op); 6060 ll_ops->fib_entry_act_ip2me_tun_pack(op_ctx, 6061 fib_entry->decap.tunnel_index); 6062 return mlxsw_sp_fib_entry_commit(mlxsw_sp, op_ctx, ll_ops); 6063 } 6064 6065 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6066 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6067 struct mlxsw_sp_fib_entry *fib_entry, 6068 enum mlxsw_sp_fib_entry_op op) 6069 { 6070 switch (fib_entry->type) { 6071 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6072 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, op_ctx, fib_entry, op); 6073 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6074 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, op_ctx, fib_entry, op); 6075 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6076 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, op_ctx, fib_entry, op); 6077 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6078 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, op_ctx, fib_entry, op); 6079 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6080 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, op_ctx, fib_entry, op); 6081 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6082 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, op_ctx, fib_entry, op); 6083 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6084 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, op_ctx, fib_entry, op); 6085 } 6086 return -EINVAL; 6087 } 6088 6089 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6090 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6091 struct mlxsw_sp_fib_entry *fib_entry, 6092 enum mlxsw_sp_fib_entry_op op) 6093 { 6094 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, op); 6095 6096 if (err) 6097 return err; 6098 6099 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6100 6101 return err; 6102 } 6103 6104 static int __mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6105 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6106 struct mlxsw_sp_fib_entry *fib_entry, 6107 bool is_new) 6108 { 6109 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 6110 is_new ? MLXSW_SP_FIB_ENTRY_OP_WRITE : 6111 MLXSW_SP_FIB_ENTRY_OP_UPDATE); 6112 } 6113 6114 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6115 struct mlxsw_sp_fib_entry *fib_entry) 6116 { 6117 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6118 6119 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6120 return __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, false); 6121 } 6122 6123 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6124 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6125 struct mlxsw_sp_fib_entry *fib_entry) 6126 { 6127 const struct mlxsw_sp_router_ll_ops *ll_ops = fib_entry->fib_node->fib->ll_ops; 6128 6129 if (!ll_ops->fib_entry_is_committed(fib_entry->priv)) 6130 return 0; 6131 return mlxsw_sp_fib_entry_op(mlxsw_sp, op_ctx, fib_entry, 6132 MLXSW_SP_FIB_ENTRY_OP_DELETE); 6133 } 6134 6135 static int 6136 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6137 const struct fib_entry_notifier_info *fen_info, 6138 struct mlxsw_sp_fib_entry *fib_entry) 6139 { 6140 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6141 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6142 struct mlxsw_sp_router *router = mlxsw_sp->router; 6143 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6144 int ifindex = nhgi->nexthops[0].ifindex; 6145 struct mlxsw_sp_ipip_entry *ipip_entry; 6146 6147 switch (fen_info->type) { 6148 case RTN_LOCAL: 6149 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6150 MLXSW_SP_L3_PROTO_IPV4, dip); 6151 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6152 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6153 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6154 fib_entry, 6155 ipip_entry); 6156 } 6157 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6158 MLXSW_SP_L3_PROTO_IPV4, 6159 &dip)) { 6160 u32 tunnel_index; 6161 6162 tunnel_index = router->nve_decap_config.tunnel_index; 6163 fib_entry->decap.tunnel_index = tunnel_index; 6164 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6165 return 0; 6166 } 6167 fallthrough; 6168 case RTN_BROADCAST: 6169 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6170 return 0; 6171 case RTN_BLACKHOLE: 6172 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6173 return 0; 6174 case RTN_UNREACHABLE: 6175 case RTN_PROHIBIT: 6176 /* Packets hitting these routes need to be trapped, but 6177 * can do so with a lower priority than packets directed 6178 * at the host, so use action type local instead of trap. 6179 */ 6180 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6181 return 0; 6182 case RTN_UNICAST: 6183 if (nhgi->gateway) 6184 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6185 else 6186 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6187 return 0; 6188 default: 6189 return -EINVAL; 6190 } 6191 } 6192 6193 static void 6194 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6195 struct mlxsw_sp_fib_entry *fib_entry) 6196 { 6197 switch (fib_entry->type) { 6198 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6199 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6200 break; 6201 default: 6202 break; 6203 } 6204 } 6205 6206 static void 6207 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6208 struct mlxsw_sp_fib4_entry *fib4_entry) 6209 { 6210 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6211 } 6212 6213 static struct mlxsw_sp_fib4_entry * 6214 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6215 struct mlxsw_sp_fib_node *fib_node, 6216 const struct fib_entry_notifier_info *fen_info) 6217 { 6218 struct mlxsw_sp_fib4_entry *fib4_entry; 6219 struct mlxsw_sp_fib_entry *fib_entry; 6220 int err; 6221 6222 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6223 if (!fib4_entry) 6224 return ERR_PTR(-ENOMEM); 6225 fib_entry = &fib4_entry->common; 6226 6227 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 6228 if (IS_ERR(fib_entry->priv)) { 6229 err = PTR_ERR(fib_entry->priv); 6230 goto err_fib_entry_priv_create; 6231 } 6232 6233 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6234 if (err) 6235 goto err_nexthop4_group_get; 6236 6237 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6238 fib_node->fib); 6239 if (err) 6240 goto err_nexthop_group_vr_link; 6241 6242 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6243 if (err) 6244 goto err_fib4_entry_type_set; 6245 6246 fib4_entry->fi = fen_info->fi; 6247 fib_info_hold(fib4_entry->fi); 6248 fib4_entry->tb_id = fen_info->tb_id; 6249 fib4_entry->type = fen_info->type; 6250 fib4_entry->dscp = fen_info->dscp; 6251 6252 fib_entry->fib_node = fib_node; 6253 6254 return fib4_entry; 6255 6256 err_fib4_entry_type_set: 6257 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6258 err_nexthop_group_vr_link: 6259 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6260 err_nexthop4_group_get: 6261 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 6262 err_fib_entry_priv_create: 6263 kfree(fib4_entry); 6264 return ERR_PTR(err); 6265 } 6266 6267 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6268 struct mlxsw_sp_fib4_entry *fib4_entry) 6269 { 6270 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6271 6272 fib_info_put(fib4_entry->fi); 6273 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6274 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6275 fib_node->fib); 6276 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6277 mlxsw_sp_fib_entry_priv_put(fib4_entry->common.priv); 6278 kfree(fib4_entry); 6279 } 6280 6281 static struct mlxsw_sp_fib4_entry * 6282 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6283 const struct fib_entry_notifier_info *fen_info) 6284 { 6285 struct mlxsw_sp_fib4_entry *fib4_entry; 6286 struct mlxsw_sp_fib_node *fib_node; 6287 struct mlxsw_sp_fib *fib; 6288 struct mlxsw_sp_vr *vr; 6289 6290 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6291 if (!vr) 6292 return NULL; 6293 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6294 6295 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6296 sizeof(fen_info->dst), 6297 fen_info->dst_len); 6298 if (!fib_node) 6299 return NULL; 6300 6301 fib4_entry = container_of(fib_node->fib_entry, 6302 struct mlxsw_sp_fib4_entry, common); 6303 if (fib4_entry->tb_id == fen_info->tb_id && 6304 fib4_entry->dscp == fen_info->dscp && 6305 fib4_entry->type == fen_info->type && 6306 fib4_entry->fi == fen_info->fi) 6307 return fib4_entry; 6308 6309 return NULL; 6310 } 6311 6312 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6313 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6314 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6315 .key_len = sizeof(struct mlxsw_sp_fib_key), 6316 .automatic_shrinking = true, 6317 }; 6318 6319 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6320 struct mlxsw_sp_fib_node *fib_node) 6321 { 6322 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6323 mlxsw_sp_fib_ht_params); 6324 } 6325 6326 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6327 struct mlxsw_sp_fib_node *fib_node) 6328 { 6329 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6330 mlxsw_sp_fib_ht_params); 6331 } 6332 6333 static struct mlxsw_sp_fib_node * 6334 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6335 size_t addr_len, unsigned char prefix_len) 6336 { 6337 struct mlxsw_sp_fib_key key; 6338 6339 memset(&key, 0, sizeof(key)); 6340 memcpy(key.addr, addr, addr_len); 6341 key.prefix_len = prefix_len; 6342 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6343 } 6344 6345 static struct mlxsw_sp_fib_node * 6346 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6347 size_t addr_len, unsigned char prefix_len) 6348 { 6349 struct mlxsw_sp_fib_node *fib_node; 6350 6351 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6352 if (!fib_node) 6353 return NULL; 6354 6355 list_add(&fib_node->list, &fib->node_list); 6356 memcpy(fib_node->key.addr, addr, addr_len); 6357 fib_node->key.prefix_len = prefix_len; 6358 6359 return fib_node; 6360 } 6361 6362 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6363 { 6364 list_del(&fib_node->list); 6365 kfree(fib_node); 6366 } 6367 6368 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6369 struct mlxsw_sp_fib_node *fib_node) 6370 { 6371 struct mlxsw_sp_prefix_usage req_prefix_usage; 6372 struct mlxsw_sp_fib *fib = fib_node->fib; 6373 struct mlxsw_sp_lpm_tree *lpm_tree; 6374 int err; 6375 6376 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6377 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6378 goto out; 6379 6380 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6381 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6382 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6383 fib->proto); 6384 if (IS_ERR(lpm_tree)) 6385 return PTR_ERR(lpm_tree); 6386 6387 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6388 if (err) 6389 goto err_lpm_tree_replace; 6390 6391 out: 6392 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6393 return 0; 6394 6395 err_lpm_tree_replace: 6396 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6397 return err; 6398 } 6399 6400 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6401 struct mlxsw_sp_fib_node *fib_node) 6402 { 6403 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6404 struct mlxsw_sp_prefix_usage req_prefix_usage; 6405 struct mlxsw_sp_fib *fib = fib_node->fib; 6406 int err; 6407 6408 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6409 return; 6410 /* Try to construct a new LPM tree from the current prefix usage 6411 * minus the unused one. If we fail, continue using the old one. 6412 */ 6413 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6414 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6415 fib_node->key.prefix_len); 6416 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6417 fib->proto); 6418 if (IS_ERR(lpm_tree)) 6419 return; 6420 6421 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6422 if (err) 6423 goto err_lpm_tree_replace; 6424 6425 return; 6426 6427 err_lpm_tree_replace: 6428 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6429 } 6430 6431 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6432 struct mlxsw_sp_fib_node *fib_node, 6433 struct mlxsw_sp_fib *fib) 6434 { 6435 int err; 6436 6437 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6438 if (err) 6439 return err; 6440 fib_node->fib = fib; 6441 6442 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6443 if (err) 6444 goto err_fib_lpm_tree_link; 6445 6446 return 0; 6447 6448 err_fib_lpm_tree_link: 6449 fib_node->fib = NULL; 6450 mlxsw_sp_fib_node_remove(fib, fib_node); 6451 return err; 6452 } 6453 6454 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6455 struct mlxsw_sp_fib_node *fib_node) 6456 { 6457 struct mlxsw_sp_fib *fib = fib_node->fib; 6458 6459 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6460 fib_node->fib = NULL; 6461 mlxsw_sp_fib_node_remove(fib, fib_node); 6462 } 6463 6464 static struct mlxsw_sp_fib_node * 6465 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6466 size_t addr_len, unsigned char prefix_len, 6467 enum mlxsw_sp_l3proto proto) 6468 { 6469 struct mlxsw_sp_fib_node *fib_node; 6470 struct mlxsw_sp_fib *fib; 6471 struct mlxsw_sp_vr *vr; 6472 int err; 6473 6474 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6475 if (IS_ERR(vr)) 6476 return ERR_CAST(vr); 6477 fib = mlxsw_sp_vr_fib(vr, proto); 6478 6479 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6480 if (fib_node) 6481 return fib_node; 6482 6483 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6484 if (!fib_node) { 6485 err = -ENOMEM; 6486 goto err_fib_node_create; 6487 } 6488 6489 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6490 if (err) 6491 goto err_fib_node_init; 6492 6493 return fib_node; 6494 6495 err_fib_node_init: 6496 mlxsw_sp_fib_node_destroy(fib_node); 6497 err_fib_node_create: 6498 mlxsw_sp_vr_put(mlxsw_sp, vr); 6499 return ERR_PTR(err); 6500 } 6501 6502 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6503 struct mlxsw_sp_fib_node *fib_node) 6504 { 6505 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6506 6507 if (fib_node->fib_entry) 6508 return; 6509 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6510 mlxsw_sp_fib_node_destroy(fib_node); 6511 mlxsw_sp_vr_put(mlxsw_sp, vr); 6512 } 6513 6514 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6515 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6516 struct mlxsw_sp_fib_entry *fib_entry) 6517 { 6518 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6519 bool is_new = !fib_node->fib_entry; 6520 int err; 6521 6522 fib_node->fib_entry = fib_entry; 6523 6524 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, fib_entry, is_new); 6525 if (err) 6526 goto err_fib_entry_update; 6527 6528 return 0; 6529 6530 err_fib_entry_update: 6531 fib_node->fib_entry = NULL; 6532 return err; 6533 } 6534 6535 static int __mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6536 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6537 struct mlxsw_sp_fib_entry *fib_entry) 6538 { 6539 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6540 int err; 6541 6542 err = mlxsw_sp_fib_entry_del(mlxsw_sp, op_ctx, fib_entry); 6543 fib_node->fib_entry = NULL; 6544 return err; 6545 } 6546 6547 static void mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6548 struct mlxsw_sp_fib_entry *fib_entry) 6549 { 6550 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = mlxsw_sp->router->ll_op_ctx; 6551 6552 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 6553 __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, fib_entry); 6554 } 6555 6556 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6557 { 6558 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6559 struct mlxsw_sp_fib4_entry *fib4_replaced; 6560 6561 if (!fib_node->fib_entry) 6562 return true; 6563 6564 fib4_replaced = container_of(fib_node->fib_entry, 6565 struct mlxsw_sp_fib4_entry, common); 6566 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6567 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6568 return false; 6569 6570 return true; 6571 } 6572 6573 static int 6574 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6575 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6576 const struct fib_entry_notifier_info *fen_info) 6577 { 6578 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6579 struct mlxsw_sp_fib_entry *replaced; 6580 struct mlxsw_sp_fib_node *fib_node; 6581 int err; 6582 6583 if (fen_info->fi->nh && 6584 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6585 return 0; 6586 6587 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6588 &fen_info->dst, sizeof(fen_info->dst), 6589 fen_info->dst_len, 6590 MLXSW_SP_L3_PROTO_IPV4); 6591 if (IS_ERR(fib_node)) { 6592 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6593 return PTR_ERR(fib_node); 6594 } 6595 6596 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6597 if (IS_ERR(fib4_entry)) { 6598 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6599 err = PTR_ERR(fib4_entry); 6600 goto err_fib4_entry_create; 6601 } 6602 6603 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6604 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6605 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6606 return 0; 6607 } 6608 6609 replaced = fib_node->fib_entry; 6610 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib4_entry->common); 6611 if (err) { 6612 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6613 goto err_fib_node_entry_link; 6614 } 6615 6616 /* Nothing to replace */ 6617 if (!replaced) 6618 return 0; 6619 6620 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6621 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6622 common); 6623 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6624 6625 return 0; 6626 6627 err_fib_node_entry_link: 6628 fib_node->fib_entry = replaced; 6629 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6630 err_fib4_entry_create: 6631 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6632 return err; 6633 } 6634 6635 static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6636 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6637 struct fib_entry_notifier_info *fen_info) 6638 { 6639 struct mlxsw_sp_fib4_entry *fib4_entry; 6640 struct mlxsw_sp_fib_node *fib_node; 6641 int err; 6642 6643 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6644 if (!fib4_entry) 6645 return 0; 6646 fib_node = fib4_entry->common.fib_node; 6647 6648 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib4_entry->common); 6649 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6650 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6651 return err; 6652 } 6653 6654 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6655 { 6656 /* Multicast routes aren't supported, so ignore them. Neighbour 6657 * Discovery packets are specifically trapped. 6658 */ 6659 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6660 return true; 6661 6662 /* Cloned routes are irrelevant in the forwarding path. */ 6663 if (rt->fib6_flags & RTF_CACHE) 6664 return true; 6665 6666 return false; 6667 } 6668 6669 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6670 { 6671 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6672 6673 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6674 if (!mlxsw_sp_rt6) 6675 return ERR_PTR(-ENOMEM); 6676 6677 /* In case of route replace, replaced route is deleted with 6678 * no notification. Take reference to prevent accessing freed 6679 * memory. 6680 */ 6681 mlxsw_sp_rt6->rt = rt; 6682 fib6_info_hold(rt); 6683 6684 return mlxsw_sp_rt6; 6685 } 6686 6687 #if IS_ENABLED(CONFIG_IPV6) 6688 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6689 { 6690 fib6_info_release(rt); 6691 } 6692 #else 6693 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6694 { 6695 } 6696 #endif 6697 6698 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6699 { 6700 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6701 6702 if (!mlxsw_sp_rt6->rt->nh) 6703 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6704 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6705 kfree(mlxsw_sp_rt6); 6706 } 6707 6708 static struct fib6_info * 6709 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6710 { 6711 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6712 list)->rt; 6713 } 6714 6715 static struct mlxsw_sp_rt6 * 6716 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6717 const struct fib6_info *rt) 6718 { 6719 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6720 6721 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6722 if (mlxsw_sp_rt6->rt == rt) 6723 return mlxsw_sp_rt6; 6724 } 6725 6726 return NULL; 6727 } 6728 6729 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6730 const struct fib6_info *rt, 6731 enum mlxsw_sp_ipip_type *ret) 6732 { 6733 return rt->fib6_nh->fib_nh_dev && 6734 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6735 } 6736 6737 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6738 struct mlxsw_sp_nexthop_group *nh_grp, 6739 struct mlxsw_sp_nexthop *nh, 6740 const struct fib6_info *rt) 6741 { 6742 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6743 6744 nh->nhgi = nh_grp->nhgi; 6745 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6746 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6747 #if IS_ENABLED(CONFIG_IPV6) 6748 nh->neigh_tbl = &nd_tbl; 6749 #endif 6750 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6751 6752 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6753 6754 if (!dev) 6755 return 0; 6756 nh->ifindex = dev->ifindex; 6757 6758 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6759 } 6760 6761 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6762 struct mlxsw_sp_nexthop *nh) 6763 { 6764 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6765 list_del(&nh->router_list_node); 6766 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6767 } 6768 6769 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6770 const struct fib6_info *rt) 6771 { 6772 return rt->fib6_nh->fib_nh_gw_family || 6773 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6774 } 6775 6776 static int 6777 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6778 struct mlxsw_sp_nexthop_group *nh_grp, 6779 struct mlxsw_sp_fib6_entry *fib6_entry) 6780 { 6781 struct mlxsw_sp_nexthop_group_info *nhgi; 6782 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6783 struct mlxsw_sp_nexthop *nh; 6784 int err, i; 6785 6786 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6787 GFP_KERNEL); 6788 if (!nhgi) 6789 return -ENOMEM; 6790 nh_grp->nhgi = nhgi; 6791 nhgi->nh_grp = nh_grp; 6792 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6793 struct mlxsw_sp_rt6, list); 6794 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6795 nhgi->count = fib6_entry->nrt6; 6796 for (i = 0; i < nhgi->count; i++) { 6797 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6798 6799 nh = &nhgi->nexthops[i]; 6800 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6801 if (err) 6802 goto err_nexthop6_init; 6803 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6804 } 6805 nh_grp->nhgi = nhgi; 6806 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6807 if (err) 6808 goto err_group_inc; 6809 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6810 if (err) 6811 goto err_group_refresh; 6812 6813 return 0; 6814 6815 err_group_refresh: 6816 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6817 err_group_inc: 6818 i = nhgi->count; 6819 err_nexthop6_init: 6820 for (i--; i >= 0; i--) { 6821 nh = &nhgi->nexthops[i]; 6822 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6823 } 6824 kfree(nhgi); 6825 return err; 6826 } 6827 6828 static void 6829 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6830 struct mlxsw_sp_nexthop_group *nh_grp) 6831 { 6832 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6833 int i; 6834 6835 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6836 for (i = nhgi->count - 1; i >= 0; i--) { 6837 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6838 6839 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6840 } 6841 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6842 WARN_ON_ONCE(nhgi->adj_index_valid); 6843 kfree(nhgi); 6844 } 6845 6846 static struct mlxsw_sp_nexthop_group * 6847 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6848 struct mlxsw_sp_fib6_entry *fib6_entry) 6849 { 6850 struct mlxsw_sp_nexthop_group *nh_grp; 6851 int err; 6852 6853 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6854 if (!nh_grp) 6855 return ERR_PTR(-ENOMEM); 6856 INIT_LIST_HEAD(&nh_grp->vr_list); 6857 err = rhashtable_init(&nh_grp->vr_ht, 6858 &mlxsw_sp_nexthop_group_vr_ht_params); 6859 if (err) 6860 goto err_nexthop_group_vr_ht_init; 6861 INIT_LIST_HEAD(&nh_grp->fib_list); 6862 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6863 6864 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6865 if (err) 6866 goto err_nexthop_group_info_init; 6867 6868 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6869 if (err) 6870 goto err_nexthop_group_insert; 6871 6872 nh_grp->can_destroy = true; 6873 6874 return nh_grp; 6875 6876 err_nexthop_group_insert: 6877 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6878 err_nexthop_group_info_init: 6879 rhashtable_destroy(&nh_grp->vr_ht); 6880 err_nexthop_group_vr_ht_init: 6881 kfree(nh_grp); 6882 return ERR_PTR(err); 6883 } 6884 6885 static void 6886 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6887 struct mlxsw_sp_nexthop_group *nh_grp) 6888 { 6889 if (!nh_grp->can_destroy) 6890 return; 6891 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6892 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6893 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6894 rhashtable_destroy(&nh_grp->vr_ht); 6895 kfree(nh_grp); 6896 } 6897 6898 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6899 struct mlxsw_sp_fib6_entry *fib6_entry) 6900 { 6901 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6902 struct mlxsw_sp_nexthop_group *nh_grp; 6903 6904 if (rt->nh) { 6905 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6906 rt->nh->id); 6907 if (WARN_ON_ONCE(!nh_grp)) 6908 return -EINVAL; 6909 goto out; 6910 } 6911 6912 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6913 if (!nh_grp) { 6914 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6915 if (IS_ERR(nh_grp)) 6916 return PTR_ERR(nh_grp); 6917 } 6918 6919 /* The route and the nexthop are described by the same struct, so we 6920 * need to the update the nexthop offload indication for the new route. 6921 */ 6922 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6923 6924 out: 6925 list_add_tail(&fib6_entry->common.nexthop_group_node, 6926 &nh_grp->fib_list); 6927 fib6_entry->common.nh_group = nh_grp; 6928 6929 return 0; 6930 } 6931 6932 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6933 struct mlxsw_sp_fib_entry *fib_entry) 6934 { 6935 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6936 6937 list_del(&fib_entry->nexthop_group_node); 6938 if (!list_empty(&nh_grp->fib_list)) 6939 return; 6940 6941 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6942 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6943 return; 6944 } 6945 6946 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6947 } 6948 6949 static int mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6950 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 6951 struct mlxsw_sp_fib6_entry *fib6_entry) 6952 { 6953 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6954 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6955 int err; 6956 6957 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6958 fib6_entry->common.nh_group = NULL; 6959 list_del(&fib6_entry->common.nexthop_group_node); 6960 6961 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6962 if (err) 6963 goto err_nexthop6_group_get; 6964 6965 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6966 fib_node->fib); 6967 if (err) 6968 goto err_nexthop_group_vr_link; 6969 6970 /* In case this entry is offloaded, then the adjacency index 6971 * currently associated with it in the device's table is that 6972 * of the old group. Start using the new one instead. 6973 */ 6974 err = __mlxsw_sp_fib_entry_update(mlxsw_sp, op_ctx, 6975 &fib6_entry->common, false); 6976 if (err) 6977 goto err_fib_entry_update; 6978 6979 if (list_empty(&old_nh_grp->fib_list)) 6980 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6981 6982 return 0; 6983 6984 err_fib_entry_update: 6985 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6986 fib_node->fib); 6987 err_nexthop_group_vr_link: 6988 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6989 err_nexthop6_group_get: 6990 list_add_tail(&fib6_entry->common.nexthop_group_node, 6991 &old_nh_grp->fib_list); 6992 fib6_entry->common.nh_group = old_nh_grp; 6993 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6994 return err; 6995 } 6996 6997 static int 6998 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6999 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7000 struct mlxsw_sp_fib6_entry *fib6_entry, 7001 struct fib6_info **rt_arr, unsigned int nrt6) 7002 { 7003 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7004 int err, i; 7005 7006 for (i = 0; i < nrt6; i++) { 7007 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7008 if (IS_ERR(mlxsw_sp_rt6)) { 7009 err = PTR_ERR(mlxsw_sp_rt6); 7010 goto err_rt6_unwind; 7011 } 7012 7013 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7014 fib6_entry->nrt6++; 7015 } 7016 7017 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 7018 if (err) 7019 goto err_rt6_unwind; 7020 7021 return 0; 7022 7023 err_rt6_unwind: 7024 for (; i > 0; i--) { 7025 fib6_entry->nrt6--; 7026 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7027 struct mlxsw_sp_rt6, list); 7028 list_del(&mlxsw_sp_rt6->list); 7029 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7030 } 7031 return err; 7032 } 7033 7034 static void 7035 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 7036 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7037 struct mlxsw_sp_fib6_entry *fib6_entry, 7038 struct fib6_info **rt_arr, unsigned int nrt6) 7039 { 7040 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7041 int i; 7042 7043 for (i = 0; i < nrt6; i++) { 7044 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 7045 rt_arr[i]); 7046 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 7047 continue; 7048 7049 fib6_entry->nrt6--; 7050 list_del(&mlxsw_sp_rt6->list); 7051 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7052 } 7053 7054 mlxsw_sp_nexthop6_group_update(mlxsw_sp, op_ctx, fib6_entry); 7055 } 7056 7057 static int 7058 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 7059 struct mlxsw_sp_fib_entry *fib_entry, 7060 const struct fib6_info *rt) 7061 { 7062 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 7063 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 7064 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 7065 struct mlxsw_sp_router *router = mlxsw_sp->router; 7066 int ifindex = nhgi->nexthops[0].ifindex; 7067 struct mlxsw_sp_ipip_entry *ipip_entry; 7068 7069 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7070 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 7071 MLXSW_SP_L3_PROTO_IPV6, 7072 dip); 7073 7074 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 7075 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 7076 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 7077 ipip_entry); 7078 } 7079 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 7080 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 7081 u32 tunnel_index; 7082 7083 tunnel_index = router->nve_decap_config.tunnel_index; 7084 fib_entry->decap.tunnel_index = tunnel_index; 7085 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 7086 } 7087 7088 return 0; 7089 } 7090 7091 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 7092 struct mlxsw_sp_fib_entry *fib_entry, 7093 const struct fib6_info *rt) 7094 { 7095 if (rt->fib6_flags & RTF_LOCAL) 7096 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 7097 rt); 7098 if (rt->fib6_flags & RTF_ANYCAST) 7099 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 7100 else if (rt->fib6_type == RTN_BLACKHOLE) 7101 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 7102 else if (rt->fib6_flags & RTF_REJECT) 7103 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 7104 else if (fib_entry->nh_group->nhgi->gateway) 7105 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7106 else 7107 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7108 7109 return 0; 7110 } 7111 7112 static void 7113 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7114 { 7115 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7116 7117 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7118 list) { 7119 fib6_entry->nrt6--; 7120 list_del(&mlxsw_sp_rt6->list); 7121 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7122 } 7123 } 7124 7125 static struct mlxsw_sp_fib6_entry * 7126 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7127 struct mlxsw_sp_fib_node *fib_node, 7128 struct fib6_info **rt_arr, unsigned int nrt6) 7129 { 7130 struct mlxsw_sp_fib6_entry *fib6_entry; 7131 struct mlxsw_sp_fib_entry *fib_entry; 7132 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7133 int err, i; 7134 7135 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 7136 if (!fib6_entry) 7137 return ERR_PTR(-ENOMEM); 7138 fib_entry = &fib6_entry->common; 7139 7140 fib_entry->priv = mlxsw_sp_fib_entry_priv_create(fib_node->fib->ll_ops); 7141 if (IS_ERR(fib_entry->priv)) { 7142 err = PTR_ERR(fib_entry->priv); 7143 goto err_fib_entry_priv_create; 7144 } 7145 7146 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7147 7148 for (i = 0; i < nrt6; i++) { 7149 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7150 if (IS_ERR(mlxsw_sp_rt6)) { 7151 err = PTR_ERR(mlxsw_sp_rt6); 7152 goto err_rt6_unwind; 7153 } 7154 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7155 fib6_entry->nrt6++; 7156 } 7157 7158 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7159 if (err) 7160 goto err_rt6_unwind; 7161 7162 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7163 fib_node->fib); 7164 if (err) 7165 goto err_nexthop_group_vr_link; 7166 7167 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7168 if (err) 7169 goto err_fib6_entry_type_set; 7170 7171 fib_entry->fib_node = fib_node; 7172 7173 return fib6_entry; 7174 7175 err_fib6_entry_type_set: 7176 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7177 err_nexthop_group_vr_link: 7178 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7179 err_rt6_unwind: 7180 for (; i > 0; i--) { 7181 fib6_entry->nrt6--; 7182 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7183 struct mlxsw_sp_rt6, list); 7184 list_del(&mlxsw_sp_rt6->list); 7185 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7186 } 7187 mlxsw_sp_fib_entry_priv_put(fib_entry->priv); 7188 err_fib_entry_priv_create: 7189 kfree(fib6_entry); 7190 return ERR_PTR(err); 7191 } 7192 7193 static void 7194 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7195 struct mlxsw_sp_fib6_entry *fib6_entry) 7196 { 7197 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7198 } 7199 7200 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7201 struct mlxsw_sp_fib6_entry *fib6_entry) 7202 { 7203 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7204 7205 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7206 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7207 fib_node->fib); 7208 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7209 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7210 WARN_ON(fib6_entry->nrt6); 7211 mlxsw_sp_fib_entry_priv_put(fib6_entry->common.priv); 7212 kfree(fib6_entry); 7213 } 7214 7215 static struct mlxsw_sp_fib6_entry * 7216 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7217 const struct fib6_info *rt) 7218 { 7219 struct mlxsw_sp_fib6_entry *fib6_entry; 7220 struct mlxsw_sp_fib_node *fib_node; 7221 struct mlxsw_sp_fib *fib; 7222 struct fib6_info *cmp_rt; 7223 struct mlxsw_sp_vr *vr; 7224 7225 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7226 if (!vr) 7227 return NULL; 7228 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7229 7230 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7231 sizeof(rt->fib6_dst.addr), 7232 rt->fib6_dst.plen); 7233 if (!fib_node) 7234 return NULL; 7235 7236 fib6_entry = container_of(fib_node->fib_entry, 7237 struct mlxsw_sp_fib6_entry, common); 7238 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7239 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7240 rt->fib6_metric == cmp_rt->fib6_metric && 7241 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7242 return fib6_entry; 7243 7244 return NULL; 7245 } 7246 7247 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7248 { 7249 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7250 struct mlxsw_sp_fib6_entry *fib6_replaced; 7251 struct fib6_info *rt, *rt_replaced; 7252 7253 if (!fib_node->fib_entry) 7254 return true; 7255 7256 fib6_replaced = container_of(fib_node->fib_entry, 7257 struct mlxsw_sp_fib6_entry, 7258 common); 7259 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7260 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7261 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7262 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7263 return false; 7264 7265 return true; 7266 } 7267 7268 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7269 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7270 struct fib6_info **rt_arr, unsigned int nrt6) 7271 { 7272 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7273 struct mlxsw_sp_fib_entry *replaced; 7274 struct mlxsw_sp_fib_node *fib_node; 7275 struct fib6_info *rt = rt_arr[0]; 7276 int err; 7277 7278 if (rt->fib6_src.plen) 7279 return -EINVAL; 7280 7281 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7282 return 0; 7283 7284 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7285 return 0; 7286 7287 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7288 &rt->fib6_dst.addr, 7289 sizeof(rt->fib6_dst.addr), 7290 rt->fib6_dst.plen, 7291 MLXSW_SP_L3_PROTO_IPV6); 7292 if (IS_ERR(fib_node)) 7293 return PTR_ERR(fib_node); 7294 7295 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7296 nrt6); 7297 if (IS_ERR(fib6_entry)) { 7298 err = PTR_ERR(fib6_entry); 7299 goto err_fib6_entry_create; 7300 } 7301 7302 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7303 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7304 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7305 return 0; 7306 } 7307 7308 replaced = fib_node->fib_entry; 7309 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, op_ctx, &fib6_entry->common); 7310 if (err) 7311 goto err_fib_node_entry_link; 7312 7313 /* Nothing to replace */ 7314 if (!replaced) 7315 return 0; 7316 7317 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7318 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7319 common); 7320 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7321 7322 return 0; 7323 7324 err_fib_node_entry_link: 7325 fib_node->fib_entry = replaced; 7326 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7327 err_fib6_entry_create: 7328 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7329 return err; 7330 } 7331 7332 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7333 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7334 struct fib6_info **rt_arr, unsigned int nrt6) 7335 { 7336 struct mlxsw_sp_fib6_entry *fib6_entry; 7337 struct mlxsw_sp_fib_node *fib_node; 7338 struct fib6_info *rt = rt_arr[0]; 7339 int err; 7340 7341 if (rt->fib6_src.plen) 7342 return -EINVAL; 7343 7344 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7345 return 0; 7346 7347 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7348 &rt->fib6_dst.addr, 7349 sizeof(rt->fib6_dst.addr), 7350 rt->fib6_dst.plen, 7351 MLXSW_SP_L3_PROTO_IPV6); 7352 if (IS_ERR(fib_node)) 7353 return PTR_ERR(fib_node); 7354 7355 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7356 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7357 return -EINVAL; 7358 } 7359 7360 fib6_entry = container_of(fib_node->fib_entry, 7361 struct mlxsw_sp_fib6_entry, common); 7362 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7363 if (err) 7364 goto err_fib6_entry_nexthop_add; 7365 7366 return 0; 7367 7368 err_fib6_entry_nexthop_add: 7369 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7370 return err; 7371 } 7372 7373 static int mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7374 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7375 struct fib6_info **rt_arr, unsigned int nrt6) 7376 { 7377 struct mlxsw_sp_fib6_entry *fib6_entry; 7378 struct mlxsw_sp_fib_node *fib_node; 7379 struct fib6_info *rt = rt_arr[0]; 7380 int err; 7381 7382 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7383 return 0; 7384 7385 /* Multipath routes are first added to the FIB trie and only then 7386 * notified. If we vetoed the addition, we will get a delete 7387 * notification for a route we do not have. Therefore, do not warn if 7388 * route was not found. 7389 */ 7390 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7391 if (!fib6_entry) 7392 return 0; 7393 7394 /* If not all the nexthops are deleted, then only reduce the nexthop 7395 * group. 7396 */ 7397 if (nrt6 != fib6_entry->nrt6) { 7398 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, op_ctx, fib6_entry, rt_arr, nrt6); 7399 return 0; 7400 } 7401 7402 fib_node = fib6_entry->common.fib_node; 7403 7404 err = __mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, op_ctx, &fib6_entry->common); 7405 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7406 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7407 return err; 7408 } 7409 7410 static struct mlxsw_sp_mr_table * 7411 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7412 { 7413 if (family == RTNL_FAMILY_IPMR) 7414 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7415 else 7416 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7417 } 7418 7419 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7420 struct mfc_entry_notifier_info *men_info, 7421 bool replace) 7422 { 7423 struct mlxsw_sp_mr_table *mrt; 7424 struct mlxsw_sp_vr *vr; 7425 7426 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7427 if (IS_ERR(vr)) 7428 return PTR_ERR(vr); 7429 7430 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7431 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7432 } 7433 7434 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7435 struct mfc_entry_notifier_info *men_info) 7436 { 7437 struct mlxsw_sp_mr_table *mrt; 7438 struct mlxsw_sp_vr *vr; 7439 7440 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7441 if (WARN_ON(!vr)) 7442 return; 7443 7444 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7445 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7446 mlxsw_sp_vr_put(mlxsw_sp, vr); 7447 } 7448 7449 static int 7450 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7451 struct vif_entry_notifier_info *ven_info) 7452 { 7453 struct mlxsw_sp_mr_table *mrt; 7454 struct mlxsw_sp_rif *rif; 7455 struct mlxsw_sp_vr *vr; 7456 7457 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7458 if (IS_ERR(vr)) 7459 return PTR_ERR(vr); 7460 7461 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7462 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7463 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7464 ven_info->vif_index, 7465 ven_info->vif_flags, rif); 7466 } 7467 7468 static void 7469 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7470 struct vif_entry_notifier_info *ven_info) 7471 { 7472 struct mlxsw_sp_mr_table *mrt; 7473 struct mlxsw_sp_vr *vr; 7474 7475 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7476 if (WARN_ON(!vr)) 7477 return; 7478 7479 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7480 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7481 mlxsw_sp_vr_put(mlxsw_sp, vr); 7482 } 7483 7484 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7485 struct mlxsw_sp_fib_node *fib_node) 7486 { 7487 struct mlxsw_sp_fib4_entry *fib4_entry; 7488 7489 fib4_entry = container_of(fib_node->fib_entry, 7490 struct mlxsw_sp_fib4_entry, common); 7491 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7492 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7493 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7494 } 7495 7496 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7497 struct mlxsw_sp_fib_node *fib_node) 7498 { 7499 struct mlxsw_sp_fib6_entry *fib6_entry; 7500 7501 fib6_entry = container_of(fib_node->fib_entry, 7502 struct mlxsw_sp_fib6_entry, common); 7503 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7504 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7505 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7506 } 7507 7508 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7509 struct mlxsw_sp_fib_node *fib_node) 7510 { 7511 switch (fib_node->fib->proto) { 7512 case MLXSW_SP_L3_PROTO_IPV4: 7513 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7514 break; 7515 case MLXSW_SP_L3_PROTO_IPV6: 7516 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7517 break; 7518 } 7519 } 7520 7521 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7522 struct mlxsw_sp_vr *vr, 7523 enum mlxsw_sp_l3proto proto) 7524 { 7525 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7526 struct mlxsw_sp_fib_node *fib_node, *tmp; 7527 7528 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7529 bool do_break = &tmp->list == &fib->node_list; 7530 7531 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7532 if (do_break) 7533 break; 7534 } 7535 } 7536 7537 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7538 { 7539 int i, j; 7540 7541 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7542 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7543 7544 if (!mlxsw_sp_vr_is_used(vr)) 7545 continue; 7546 7547 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7548 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7549 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7550 7551 /* If virtual router was only used for IPv4, then it's no 7552 * longer used. 7553 */ 7554 if (!mlxsw_sp_vr_is_used(vr)) 7555 continue; 7556 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7557 } 7558 } 7559 7560 struct mlxsw_sp_fib6_event { 7561 struct fib6_info **rt_arr; 7562 unsigned int nrt6; 7563 }; 7564 7565 struct mlxsw_sp_fib_event { 7566 struct list_head list; /* node in fib queue */ 7567 union { 7568 struct mlxsw_sp_fib6_event fib6_event; 7569 struct fib_entry_notifier_info fen_info; 7570 struct fib_rule_notifier_info fr_info; 7571 struct fib_nh_notifier_info fnh_info; 7572 struct mfc_entry_notifier_info men_info; 7573 struct vif_entry_notifier_info ven_info; 7574 }; 7575 struct mlxsw_sp *mlxsw_sp; 7576 unsigned long event; 7577 int family; 7578 }; 7579 7580 static int 7581 mlxsw_sp_router_fib6_event_init(struct mlxsw_sp_fib6_event *fib6_event, 7582 struct fib6_entry_notifier_info *fen6_info) 7583 { 7584 struct fib6_info *rt = fen6_info->rt; 7585 struct fib6_info **rt_arr; 7586 struct fib6_info *iter; 7587 unsigned int nrt6; 7588 int i = 0; 7589 7590 nrt6 = fen6_info->nsiblings + 1; 7591 7592 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7593 if (!rt_arr) 7594 return -ENOMEM; 7595 7596 fib6_event->rt_arr = rt_arr; 7597 fib6_event->nrt6 = nrt6; 7598 7599 rt_arr[0] = rt; 7600 fib6_info_hold(rt); 7601 7602 if (!fen6_info->nsiblings) 7603 return 0; 7604 7605 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7606 if (i == fen6_info->nsiblings) 7607 break; 7608 7609 rt_arr[i + 1] = iter; 7610 fib6_info_hold(iter); 7611 i++; 7612 } 7613 WARN_ON_ONCE(i != fen6_info->nsiblings); 7614 7615 return 0; 7616 } 7617 7618 static void 7619 mlxsw_sp_router_fib6_event_fini(struct mlxsw_sp_fib6_event *fib6_event) 7620 { 7621 int i; 7622 7623 for (i = 0; i < fib6_event->nrt6; i++) 7624 mlxsw_sp_rt6_release(fib6_event->rt_arr[i]); 7625 kfree(fib6_event->rt_arr); 7626 } 7627 7628 static void mlxsw_sp_router_fib4_event_process(struct mlxsw_sp *mlxsw_sp, 7629 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7630 struct mlxsw_sp_fib_event *fib_event) 7631 { 7632 int err; 7633 7634 mlxsw_sp_span_respin(mlxsw_sp); 7635 7636 switch (fib_event->event) { 7637 case FIB_EVENT_ENTRY_REPLACE: 7638 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, op_ctx, &fib_event->fen_info); 7639 if (err) { 7640 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7641 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7642 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7643 &fib_event->fen_info); 7644 } 7645 fib_info_put(fib_event->fen_info.fi); 7646 break; 7647 case FIB_EVENT_ENTRY_DEL: 7648 err = mlxsw_sp_router_fib4_del(mlxsw_sp, op_ctx, &fib_event->fen_info); 7649 if (err) 7650 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7651 fib_info_put(fib_event->fen_info.fi); 7652 break; 7653 case FIB_EVENT_NH_ADD: 7654 case FIB_EVENT_NH_DEL: 7655 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_event->event, fib_event->fnh_info.fib_nh); 7656 fib_info_put(fib_event->fnh_info.fib_nh->nh_parent); 7657 break; 7658 } 7659 } 7660 7661 static void mlxsw_sp_router_fib6_event_process(struct mlxsw_sp *mlxsw_sp, 7662 struct mlxsw_sp_fib_entry_op_ctx *op_ctx, 7663 struct mlxsw_sp_fib_event *fib_event) 7664 { 7665 struct mlxsw_sp_fib6_event *fib6_event = &fib_event->fib6_event; 7666 int err; 7667 7668 mlxsw_sp_span_respin(mlxsw_sp); 7669 7670 switch (fib_event->event) { 7671 case FIB_EVENT_ENTRY_REPLACE: 7672 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7673 fib_event->fib6_event.nrt6); 7674 if (err) { 7675 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7676 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7677 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7678 fib6_event->rt_arr, 7679 fib6_event->nrt6); 7680 } 7681 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7682 break; 7683 case FIB_EVENT_ENTRY_APPEND: 7684 err = mlxsw_sp_router_fib6_append(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7685 fib_event->fib6_event.nrt6); 7686 if (err) { 7687 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7688 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7689 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7690 fib6_event->rt_arr, 7691 fib6_event->nrt6); 7692 } 7693 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7694 break; 7695 case FIB_EVENT_ENTRY_DEL: 7696 err = mlxsw_sp_router_fib6_del(mlxsw_sp, op_ctx, fib_event->fib6_event.rt_arr, 7697 fib_event->fib6_event.nrt6); 7698 if (err) 7699 mlxsw_sp_fib_entry_op_ctx_priv_put_all(op_ctx); 7700 mlxsw_sp_router_fib6_event_fini(&fib_event->fib6_event); 7701 break; 7702 } 7703 } 7704 7705 static void mlxsw_sp_router_fibmr_event_process(struct mlxsw_sp *mlxsw_sp, 7706 struct mlxsw_sp_fib_event *fib_event) 7707 { 7708 bool replace; 7709 int err; 7710 7711 rtnl_lock(); 7712 mutex_lock(&mlxsw_sp->router->lock); 7713 switch (fib_event->event) { 7714 case FIB_EVENT_ENTRY_REPLACE: 7715 case FIB_EVENT_ENTRY_ADD: 7716 replace = fib_event->event == FIB_EVENT_ENTRY_REPLACE; 7717 7718 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_event->men_info, replace); 7719 if (err) 7720 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7721 mr_cache_put(fib_event->men_info.mfc); 7722 break; 7723 case FIB_EVENT_ENTRY_DEL: 7724 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_event->men_info); 7725 mr_cache_put(fib_event->men_info.mfc); 7726 break; 7727 case FIB_EVENT_VIF_ADD: 7728 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7729 &fib_event->ven_info); 7730 if (err) 7731 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7732 dev_put(fib_event->ven_info.dev); 7733 break; 7734 case FIB_EVENT_VIF_DEL: 7735 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, &fib_event->ven_info); 7736 dev_put(fib_event->ven_info.dev); 7737 break; 7738 } 7739 mutex_unlock(&mlxsw_sp->router->lock); 7740 rtnl_unlock(); 7741 } 7742 7743 static void mlxsw_sp_router_fib_event_work(struct work_struct *work) 7744 { 7745 struct mlxsw_sp_router *router = container_of(work, struct mlxsw_sp_router, fib_event_work); 7746 struct mlxsw_sp_fib_entry_op_ctx *op_ctx = router->ll_op_ctx; 7747 struct mlxsw_sp *mlxsw_sp = router->mlxsw_sp; 7748 struct mlxsw_sp_fib_event *next_fib_event; 7749 struct mlxsw_sp_fib_event *fib_event; 7750 int last_family = AF_UNSPEC; 7751 LIST_HEAD(fib_event_queue); 7752 7753 spin_lock_bh(&router->fib_event_queue_lock); 7754 list_splice_init(&router->fib_event_queue, &fib_event_queue); 7755 spin_unlock_bh(&router->fib_event_queue_lock); 7756 7757 /* Router lock is held here to make sure per-instance 7758 * operation context is not used in between FIB4/6 events 7759 * processing. 7760 */ 7761 mutex_lock(&router->lock); 7762 mlxsw_sp_fib_entry_op_ctx_clear(op_ctx); 7763 list_for_each_entry_safe(fib_event, next_fib_event, 7764 &fib_event_queue, list) { 7765 /* Check if the next entry in the queue exists and it is 7766 * of the same type (family and event) as the currect one. 7767 * In that case it is permitted to do the bulking 7768 * of multiple FIB entries to a single register write. 7769 */ 7770 op_ctx->bulk_ok = !list_is_last(&fib_event->list, &fib_event_queue) && 7771 fib_event->family == next_fib_event->family && 7772 fib_event->event == next_fib_event->event; 7773 op_ctx->event = fib_event->event; 7774 7775 /* In case family of this and the previous entry are different, context 7776 * reinitialization is going to be needed now, indicate that. 7777 * Note that since last_family is initialized to AF_UNSPEC, this is always 7778 * going to happen for the first entry processed in the work. 7779 */ 7780 if (fib_event->family != last_family) 7781 op_ctx->initialized = false; 7782 7783 switch (fib_event->family) { 7784 case AF_INET: 7785 mlxsw_sp_router_fib4_event_process(mlxsw_sp, op_ctx, 7786 fib_event); 7787 break; 7788 case AF_INET6: 7789 mlxsw_sp_router_fib6_event_process(mlxsw_sp, op_ctx, 7790 fib_event); 7791 break; 7792 case RTNL_FAMILY_IP6MR: 7793 case RTNL_FAMILY_IPMR: 7794 /* Unlock here as inside FIBMR the lock is taken again 7795 * under RTNL. The per-instance operation context 7796 * is not used by FIBMR. 7797 */ 7798 mutex_unlock(&router->lock); 7799 mlxsw_sp_router_fibmr_event_process(mlxsw_sp, 7800 fib_event); 7801 mutex_lock(&router->lock); 7802 break; 7803 default: 7804 WARN_ON_ONCE(1); 7805 } 7806 last_family = fib_event->family; 7807 kfree(fib_event); 7808 cond_resched(); 7809 } 7810 WARN_ON_ONCE(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 7811 mutex_unlock(&router->lock); 7812 } 7813 7814 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event *fib_event, 7815 struct fib_notifier_info *info) 7816 { 7817 struct fib_entry_notifier_info *fen_info; 7818 struct fib_nh_notifier_info *fnh_info; 7819 7820 switch (fib_event->event) { 7821 case FIB_EVENT_ENTRY_REPLACE: 7822 case FIB_EVENT_ENTRY_DEL: 7823 fen_info = container_of(info, struct fib_entry_notifier_info, 7824 info); 7825 fib_event->fen_info = *fen_info; 7826 /* Take reference on fib_info to prevent it from being 7827 * freed while event is queued. Release it afterwards. 7828 */ 7829 fib_info_hold(fib_event->fen_info.fi); 7830 break; 7831 case FIB_EVENT_NH_ADD: 7832 case FIB_EVENT_NH_DEL: 7833 fnh_info = container_of(info, struct fib_nh_notifier_info, 7834 info); 7835 fib_event->fnh_info = *fnh_info; 7836 fib_info_hold(fib_event->fnh_info.fib_nh->nh_parent); 7837 break; 7838 } 7839 } 7840 7841 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event *fib_event, 7842 struct fib_notifier_info *info) 7843 { 7844 struct fib6_entry_notifier_info *fen6_info; 7845 int err; 7846 7847 switch (fib_event->event) { 7848 case FIB_EVENT_ENTRY_REPLACE: 7849 case FIB_EVENT_ENTRY_APPEND: 7850 case FIB_EVENT_ENTRY_DEL: 7851 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7852 info); 7853 err = mlxsw_sp_router_fib6_event_init(&fib_event->fib6_event, 7854 fen6_info); 7855 if (err) 7856 return err; 7857 break; 7858 } 7859 7860 return 0; 7861 } 7862 7863 static void 7864 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event *fib_event, 7865 struct fib_notifier_info *info) 7866 { 7867 switch (fib_event->event) { 7868 case FIB_EVENT_ENTRY_REPLACE: 7869 case FIB_EVENT_ENTRY_ADD: 7870 case FIB_EVENT_ENTRY_DEL: 7871 memcpy(&fib_event->men_info, info, sizeof(fib_event->men_info)); 7872 mr_cache_hold(fib_event->men_info.mfc); 7873 break; 7874 case FIB_EVENT_VIF_ADD: 7875 case FIB_EVENT_VIF_DEL: 7876 memcpy(&fib_event->ven_info, info, sizeof(fib_event->ven_info)); 7877 dev_hold(fib_event->ven_info.dev); 7878 break; 7879 } 7880 } 7881 7882 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7883 struct fib_notifier_info *info, 7884 struct mlxsw_sp *mlxsw_sp) 7885 { 7886 struct netlink_ext_ack *extack = info->extack; 7887 struct fib_rule_notifier_info *fr_info; 7888 struct fib_rule *rule; 7889 int err = 0; 7890 7891 /* nothing to do at the moment */ 7892 if (event == FIB_EVENT_RULE_DEL) 7893 return 0; 7894 7895 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7896 rule = fr_info->rule; 7897 7898 /* Rule only affects locally generated traffic */ 7899 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7900 return 0; 7901 7902 switch (info->family) { 7903 case AF_INET: 7904 if (!fib4_rule_default(rule) && !rule->l3mdev) 7905 err = -EOPNOTSUPP; 7906 break; 7907 case AF_INET6: 7908 if (!fib6_rule_default(rule) && !rule->l3mdev) 7909 err = -EOPNOTSUPP; 7910 break; 7911 case RTNL_FAMILY_IPMR: 7912 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7913 err = -EOPNOTSUPP; 7914 break; 7915 case RTNL_FAMILY_IP6MR: 7916 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7917 err = -EOPNOTSUPP; 7918 break; 7919 } 7920 7921 if (err < 0) 7922 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7923 7924 return err; 7925 } 7926 7927 /* Called with rcu_read_lock() */ 7928 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7929 unsigned long event, void *ptr) 7930 { 7931 struct mlxsw_sp_fib_event *fib_event; 7932 struct fib_notifier_info *info = ptr; 7933 struct mlxsw_sp_router *router; 7934 int err; 7935 7936 if ((info->family != AF_INET && info->family != AF_INET6 && 7937 info->family != RTNL_FAMILY_IPMR && 7938 info->family != RTNL_FAMILY_IP6MR)) 7939 return NOTIFY_DONE; 7940 7941 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7942 7943 switch (event) { 7944 case FIB_EVENT_RULE_ADD: 7945 case FIB_EVENT_RULE_DEL: 7946 err = mlxsw_sp_router_fib_rule_event(event, info, 7947 router->mlxsw_sp); 7948 return notifier_from_errno(err); 7949 case FIB_EVENT_ENTRY_ADD: 7950 case FIB_EVENT_ENTRY_REPLACE: 7951 case FIB_EVENT_ENTRY_APPEND: 7952 if (info->family == AF_INET) { 7953 struct fib_entry_notifier_info *fen_info = ptr; 7954 7955 if (fen_info->fi->fib_nh_is_v6) { 7956 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7957 return notifier_from_errno(-EINVAL); 7958 } 7959 } 7960 break; 7961 } 7962 7963 fib_event = kzalloc(sizeof(*fib_event), GFP_ATOMIC); 7964 if (!fib_event) 7965 return NOTIFY_BAD; 7966 7967 fib_event->mlxsw_sp = router->mlxsw_sp; 7968 fib_event->event = event; 7969 fib_event->family = info->family; 7970 7971 switch (info->family) { 7972 case AF_INET: 7973 mlxsw_sp_router_fib4_event(fib_event, info); 7974 break; 7975 case AF_INET6: 7976 err = mlxsw_sp_router_fib6_event(fib_event, info); 7977 if (err) 7978 goto err_fib_event; 7979 break; 7980 case RTNL_FAMILY_IP6MR: 7981 case RTNL_FAMILY_IPMR: 7982 mlxsw_sp_router_fibmr_event(fib_event, info); 7983 break; 7984 } 7985 7986 /* Enqueue the event and trigger the work */ 7987 spin_lock_bh(&router->fib_event_queue_lock); 7988 list_add_tail(&fib_event->list, &router->fib_event_queue); 7989 spin_unlock_bh(&router->fib_event_queue_lock); 7990 mlxsw_core_schedule_work(&router->fib_event_work); 7991 7992 return NOTIFY_DONE; 7993 7994 err_fib_event: 7995 kfree(fib_event); 7996 return NOTIFY_BAD; 7997 } 7998 7999 static struct mlxsw_sp_rif * 8000 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 8001 const struct net_device *dev) 8002 { 8003 int i; 8004 8005 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 8006 if (mlxsw_sp->router->rifs[i] && 8007 mlxsw_sp->router->rifs[i]->dev == dev) 8008 return mlxsw_sp->router->rifs[i]; 8009 8010 return NULL; 8011 } 8012 8013 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 8014 const struct net_device *dev) 8015 { 8016 struct mlxsw_sp_rif *rif; 8017 8018 mutex_lock(&mlxsw_sp->router->lock); 8019 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8020 mutex_unlock(&mlxsw_sp->router->lock); 8021 8022 return rif; 8023 } 8024 8025 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 8026 { 8027 struct mlxsw_sp_rif *rif; 8028 u16 vid = 0; 8029 8030 mutex_lock(&mlxsw_sp->router->lock); 8031 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8032 if (!rif) 8033 goto out; 8034 8035 /* We only return the VID for VLAN RIFs. Otherwise we return an 8036 * invalid value (0). 8037 */ 8038 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 8039 goto out; 8040 8041 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 8042 8043 out: 8044 mutex_unlock(&mlxsw_sp->router->lock); 8045 return vid; 8046 } 8047 8048 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 8049 { 8050 char ritr_pl[MLXSW_REG_RITR_LEN]; 8051 int err; 8052 8053 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 8054 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8055 if (err) 8056 return err; 8057 8058 mlxsw_reg_ritr_enable_set(ritr_pl, false); 8059 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8060 } 8061 8062 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 8063 struct mlxsw_sp_rif *rif) 8064 { 8065 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 8066 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 8067 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 8068 } 8069 8070 static bool 8071 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 8072 unsigned long event) 8073 { 8074 struct inet6_dev *inet6_dev; 8075 bool addr_list_empty = true; 8076 struct in_device *idev; 8077 8078 switch (event) { 8079 case NETDEV_UP: 8080 return rif == NULL; 8081 case NETDEV_DOWN: 8082 rcu_read_lock(); 8083 idev = __in_dev_get_rcu(dev); 8084 if (idev && idev->ifa_list) 8085 addr_list_empty = false; 8086 8087 inet6_dev = __in6_dev_get(dev); 8088 if (addr_list_empty && inet6_dev && 8089 !list_empty(&inet6_dev->addr_list)) 8090 addr_list_empty = false; 8091 rcu_read_unlock(); 8092 8093 /* macvlans do not have a RIF, but rather piggy back on the 8094 * RIF of their lower device. 8095 */ 8096 if (netif_is_macvlan(dev) && addr_list_empty) 8097 return true; 8098 8099 if (rif && addr_list_empty && 8100 !netif_is_l3_slave(rif->dev)) 8101 return true; 8102 /* It is possible we already removed the RIF ourselves 8103 * if it was assigned to a netdev that is now a bridge 8104 * or LAG slave. 8105 */ 8106 return false; 8107 } 8108 8109 return false; 8110 } 8111 8112 static enum mlxsw_sp_rif_type 8113 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 8114 const struct net_device *dev) 8115 { 8116 enum mlxsw_sp_fid_type type; 8117 8118 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 8119 return MLXSW_SP_RIF_TYPE_IPIP_LB; 8120 8121 /* Otherwise RIF type is derived from the type of the underlying FID. */ 8122 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 8123 type = MLXSW_SP_FID_TYPE_8021Q; 8124 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 8125 type = MLXSW_SP_FID_TYPE_8021Q; 8126 else if (netif_is_bridge_master(dev)) 8127 type = MLXSW_SP_FID_TYPE_8021D; 8128 else 8129 type = MLXSW_SP_FID_TYPE_RFID; 8130 8131 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 8132 } 8133 8134 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 8135 { 8136 int i; 8137 8138 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 8139 if (!mlxsw_sp->router->rifs[i]) { 8140 *p_rif_index = i; 8141 return 0; 8142 } 8143 } 8144 8145 return -ENOBUFS; 8146 } 8147 8148 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8149 u16 vr_id, 8150 struct net_device *l3_dev) 8151 { 8152 struct mlxsw_sp_rif *rif; 8153 8154 rif = kzalloc(rif_size, GFP_KERNEL); 8155 if (!rif) 8156 return NULL; 8157 8158 INIT_LIST_HEAD(&rif->nexthop_list); 8159 INIT_LIST_HEAD(&rif->neigh_list); 8160 if (l3_dev) { 8161 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8162 rif->mtu = l3_dev->mtu; 8163 rif->dev = l3_dev; 8164 } 8165 rif->vr_id = vr_id; 8166 rif->rif_index = rif_index; 8167 8168 return rif; 8169 } 8170 8171 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8172 u16 rif_index) 8173 { 8174 return mlxsw_sp->router->rifs[rif_index]; 8175 } 8176 8177 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8178 { 8179 return rif->rif_index; 8180 } 8181 8182 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8183 { 8184 return lb_rif->common.rif_index; 8185 } 8186 8187 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8188 { 8189 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 8190 struct mlxsw_sp_vr *ul_vr; 8191 8192 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8193 if (WARN_ON(IS_ERR(ul_vr))) 8194 return 0; 8195 8196 return ul_vr->id; 8197 } 8198 8199 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8200 { 8201 return lb_rif->ul_rif_id; 8202 } 8203 8204 static bool 8205 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8206 { 8207 return mlxsw_sp_rif_counter_valid_get(rif, 8208 MLXSW_SP_RIF_COUNTER_EGRESS) && 8209 mlxsw_sp_rif_counter_valid_get(rif, 8210 MLXSW_SP_RIF_COUNTER_INGRESS); 8211 } 8212 8213 static int 8214 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8215 { 8216 int err; 8217 8218 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8219 if (err) 8220 return err; 8221 8222 /* Clear stale data. */ 8223 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8224 MLXSW_SP_RIF_COUNTER_INGRESS, 8225 NULL); 8226 if (err) 8227 goto err_clear_ingress; 8228 8229 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8230 if (err) 8231 goto err_alloc_egress; 8232 8233 /* Clear stale data. */ 8234 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8235 MLXSW_SP_RIF_COUNTER_EGRESS, 8236 NULL); 8237 if (err) 8238 goto err_clear_egress; 8239 8240 return 0; 8241 8242 err_clear_egress: 8243 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8244 err_alloc_egress: 8245 err_clear_ingress: 8246 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8247 return err; 8248 } 8249 8250 static void 8251 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8252 { 8253 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8254 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8255 } 8256 8257 static void 8258 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8259 struct netdev_notifier_offload_xstats_info *info) 8260 { 8261 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8262 return; 8263 netdev_offload_xstats_report_used(info->report_used); 8264 } 8265 8266 static int 8267 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8268 struct rtnl_hw_stats64 *p_stats) 8269 { 8270 struct mlxsw_sp_rif_counter_set_basic ingress; 8271 struct mlxsw_sp_rif_counter_set_basic egress; 8272 int err; 8273 8274 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8275 MLXSW_SP_RIF_COUNTER_INGRESS, 8276 &ingress); 8277 if (err) 8278 return err; 8279 8280 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8281 MLXSW_SP_RIF_COUNTER_EGRESS, 8282 &egress); 8283 if (err) 8284 return err; 8285 8286 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8287 ((SET.good_unicast_ ## SFX) + \ 8288 (SET.good_multicast_ ## SFX) + \ 8289 (SET.good_broadcast_ ## SFX)) 8290 8291 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8292 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8293 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8294 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8295 p_stats->rx_errors = ingress.error_packets; 8296 p_stats->tx_errors = egress.error_packets; 8297 p_stats->rx_dropped = ingress.discard_packets; 8298 p_stats->tx_dropped = egress.discard_packets; 8299 p_stats->multicast = ingress.good_multicast_packets + 8300 ingress.good_broadcast_packets; 8301 8302 #undef MLXSW_SP_ROUTER_ALL_GOOD 8303 8304 return 0; 8305 } 8306 8307 static int 8308 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8309 struct netdev_notifier_offload_xstats_info *info) 8310 { 8311 struct rtnl_hw_stats64 stats = {}; 8312 int err; 8313 8314 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8315 return 0; 8316 8317 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8318 if (err) 8319 return err; 8320 8321 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8322 return 0; 8323 } 8324 8325 struct mlxsw_sp_router_hwstats_notify_work { 8326 struct work_struct work; 8327 struct net_device *dev; 8328 }; 8329 8330 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8331 { 8332 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8333 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8334 work); 8335 8336 rtnl_lock(); 8337 rtnl_offload_xstats_notify(hws_work->dev); 8338 rtnl_unlock(); 8339 dev_put(hws_work->dev); 8340 kfree(hws_work); 8341 } 8342 8343 static void 8344 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8345 { 8346 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8347 8348 /* To collect notification payload, the core ends up sending another 8349 * notifier block message, which would deadlock on the attempt to 8350 * acquire the router lock again. Just postpone the notification until 8351 * later. 8352 */ 8353 8354 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8355 if (!hws_work) 8356 return; 8357 8358 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8359 dev_hold(dev); 8360 hws_work->dev = dev; 8361 mlxsw_core_schedule_work(&hws_work->work); 8362 } 8363 8364 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8365 { 8366 return rif->dev->ifindex; 8367 } 8368 8369 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 8370 { 8371 return rif->dev; 8372 } 8373 8374 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8375 { 8376 struct rtnl_hw_stats64 stats = {}; 8377 8378 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8379 netdev_offload_xstats_push_delta(rif->dev, 8380 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8381 &stats); 8382 } 8383 8384 static struct mlxsw_sp_rif * 8385 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8386 const struct mlxsw_sp_rif_params *params, 8387 struct netlink_ext_ack *extack) 8388 { 8389 u32 tb_id = l3mdev_fib_table(params->dev); 8390 const struct mlxsw_sp_rif_ops *ops; 8391 struct mlxsw_sp_fid *fid = NULL; 8392 enum mlxsw_sp_rif_type type; 8393 struct mlxsw_sp_rif *rif; 8394 struct mlxsw_sp_vr *vr; 8395 u16 rif_index; 8396 int i, err; 8397 8398 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8399 ops = mlxsw_sp->router->rif_ops_arr[type]; 8400 8401 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8402 if (IS_ERR(vr)) 8403 return ERR_CAST(vr); 8404 vr->rif_count++; 8405 8406 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8407 if (err) { 8408 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8409 goto err_rif_index_alloc; 8410 } 8411 8412 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 8413 if (!rif) { 8414 err = -ENOMEM; 8415 goto err_rif_alloc; 8416 } 8417 dev_hold(rif->dev); 8418 mlxsw_sp->router->rifs[rif_index] = rif; 8419 rif->mlxsw_sp = mlxsw_sp; 8420 rif->ops = ops; 8421 8422 if (ops->fid_get) { 8423 fid = ops->fid_get(rif, extack); 8424 if (IS_ERR(fid)) { 8425 err = PTR_ERR(fid); 8426 goto err_fid_get; 8427 } 8428 rif->fid = fid; 8429 } 8430 8431 if (ops->setup) 8432 ops->setup(rif, params); 8433 8434 err = ops->configure(rif, extack); 8435 if (err) 8436 goto err_configure; 8437 8438 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8439 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8440 if (err) 8441 goto err_mr_rif_add; 8442 } 8443 8444 if (netdev_offload_xstats_enabled(rif->dev, 8445 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8446 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8447 if (err) 8448 goto err_stats_enable; 8449 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8450 } else { 8451 mlxsw_sp_rif_counters_alloc(rif); 8452 } 8453 8454 return rif; 8455 8456 err_stats_enable: 8457 err_mr_rif_add: 8458 for (i--; i >= 0; i--) 8459 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8460 ops->deconfigure(rif); 8461 err_configure: 8462 if (fid) 8463 mlxsw_sp_fid_put(fid); 8464 err_fid_get: 8465 mlxsw_sp->router->rifs[rif_index] = NULL; 8466 dev_put(rif->dev); 8467 kfree(rif); 8468 err_rif_alloc: 8469 err_rif_index_alloc: 8470 vr->rif_count--; 8471 mlxsw_sp_vr_put(mlxsw_sp, vr); 8472 return ERR_PTR(err); 8473 } 8474 8475 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8476 { 8477 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8478 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8479 struct mlxsw_sp_fid *fid = rif->fid; 8480 struct mlxsw_sp_vr *vr; 8481 int i; 8482 8483 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8484 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8485 8486 if (netdev_offload_xstats_enabled(rif->dev, 8487 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8488 mlxsw_sp_rif_push_l3_stats(rif); 8489 mlxsw_sp_router_port_l3_stats_disable(rif); 8490 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8491 } else { 8492 mlxsw_sp_rif_counters_free(rif); 8493 } 8494 8495 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8496 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8497 ops->deconfigure(rif); 8498 if (fid) 8499 /* Loopback RIFs are not associated with a FID. */ 8500 mlxsw_sp_fid_put(fid); 8501 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8502 dev_put(rif->dev); 8503 kfree(rif); 8504 vr->rif_count--; 8505 mlxsw_sp_vr_put(mlxsw_sp, vr); 8506 } 8507 8508 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8509 struct net_device *dev) 8510 { 8511 struct mlxsw_sp_rif *rif; 8512 8513 mutex_lock(&mlxsw_sp->router->lock); 8514 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8515 if (!rif) 8516 goto out; 8517 mlxsw_sp_rif_destroy(rif); 8518 out: 8519 mutex_unlock(&mlxsw_sp->router->lock); 8520 } 8521 8522 static void 8523 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8524 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8525 { 8526 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8527 8528 params->vid = mlxsw_sp_port_vlan->vid; 8529 params->lag = mlxsw_sp_port->lagged; 8530 if (params->lag) 8531 params->lag_id = mlxsw_sp_port->lag_id; 8532 else 8533 params->system_port = mlxsw_sp_port->local_port; 8534 } 8535 8536 static struct mlxsw_sp_rif_subport * 8537 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8538 { 8539 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8540 } 8541 8542 static struct mlxsw_sp_rif * 8543 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8544 const struct mlxsw_sp_rif_params *params, 8545 struct netlink_ext_ack *extack) 8546 { 8547 struct mlxsw_sp_rif_subport *rif_subport; 8548 struct mlxsw_sp_rif *rif; 8549 8550 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8551 if (!rif) 8552 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8553 8554 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8555 refcount_inc(&rif_subport->ref_count); 8556 return rif; 8557 } 8558 8559 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8560 { 8561 struct mlxsw_sp_rif_subport *rif_subport; 8562 8563 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8564 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8565 return; 8566 8567 mlxsw_sp_rif_destroy(rif); 8568 } 8569 8570 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8571 struct mlxsw_sp_rif_mac_profile *profile, 8572 struct netlink_ext_ack *extack) 8573 { 8574 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8575 struct mlxsw_sp_router *router = mlxsw_sp->router; 8576 int id; 8577 8578 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8579 max_rif_mac_profiles, GFP_KERNEL); 8580 8581 if (id >= 0) { 8582 profile->id = id; 8583 return 0; 8584 } 8585 8586 if (id == -ENOSPC) 8587 NL_SET_ERR_MSG_MOD(extack, 8588 "Exceeded number of supported router interface MAC profiles"); 8589 8590 return id; 8591 } 8592 8593 static struct mlxsw_sp_rif_mac_profile * 8594 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8595 { 8596 struct mlxsw_sp_rif_mac_profile *profile; 8597 8598 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8599 mac_profile); 8600 WARN_ON(!profile); 8601 return profile; 8602 } 8603 8604 static struct mlxsw_sp_rif_mac_profile * 8605 mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8606 { 8607 struct mlxsw_sp_rif_mac_profile *profile; 8608 8609 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8610 if (!profile) 8611 return NULL; 8612 8613 ether_addr_copy(profile->mac_prefix, mac); 8614 refcount_set(&profile->ref_count, 1); 8615 return profile; 8616 } 8617 8618 static struct mlxsw_sp_rif_mac_profile * 8619 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8620 { 8621 struct mlxsw_sp_router *router = mlxsw_sp->router; 8622 struct mlxsw_sp_rif_mac_profile *profile; 8623 int id; 8624 8625 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8626 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8627 mlxsw_sp->mac_mask)) 8628 return profile; 8629 } 8630 8631 return NULL; 8632 } 8633 8634 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8635 { 8636 const struct mlxsw_sp *mlxsw_sp = priv; 8637 8638 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8639 } 8640 8641 static struct mlxsw_sp_rif_mac_profile * 8642 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8643 struct netlink_ext_ack *extack) 8644 { 8645 struct mlxsw_sp_rif_mac_profile *profile; 8646 int err; 8647 8648 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8649 if (!profile) 8650 return ERR_PTR(-ENOMEM); 8651 8652 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8653 if (err) 8654 goto profile_index_alloc_err; 8655 8656 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8657 return profile; 8658 8659 profile_index_alloc_err: 8660 kfree(profile); 8661 return ERR_PTR(err); 8662 } 8663 8664 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8665 u8 mac_profile) 8666 { 8667 struct mlxsw_sp_rif_mac_profile *profile; 8668 8669 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8670 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8671 kfree(profile); 8672 } 8673 8674 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8675 const char *mac, u8 *p_mac_profile, 8676 struct netlink_ext_ack *extack) 8677 { 8678 struct mlxsw_sp_rif_mac_profile *profile; 8679 8680 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8681 if (profile) { 8682 refcount_inc(&profile->ref_count); 8683 goto out; 8684 } 8685 8686 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8687 if (IS_ERR(profile)) 8688 return PTR_ERR(profile); 8689 8690 out: 8691 *p_mac_profile = profile->id; 8692 return 0; 8693 } 8694 8695 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8696 u8 mac_profile) 8697 { 8698 struct mlxsw_sp_rif_mac_profile *profile; 8699 8700 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8701 mac_profile); 8702 if (WARN_ON(!profile)) 8703 return; 8704 8705 if (!refcount_dec_and_test(&profile->ref_count)) 8706 return; 8707 8708 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8709 } 8710 8711 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8712 { 8713 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8714 struct mlxsw_sp_rif_mac_profile *profile; 8715 8716 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8717 rif->mac_profile_id); 8718 if (WARN_ON(!profile)) 8719 return false; 8720 8721 return refcount_read(&profile->ref_count) > 1; 8722 } 8723 8724 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8725 const char *new_mac) 8726 { 8727 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8728 struct mlxsw_sp_rif_mac_profile *profile; 8729 8730 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8731 rif->mac_profile_id); 8732 if (WARN_ON(!profile)) 8733 return -EINVAL; 8734 8735 ether_addr_copy(profile->mac_prefix, new_mac); 8736 return 0; 8737 } 8738 8739 static int 8740 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8741 struct mlxsw_sp_rif *rif, 8742 const char *new_mac, 8743 struct netlink_ext_ack *extack) 8744 { 8745 u8 mac_profile; 8746 int err; 8747 8748 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8749 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8750 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8751 8752 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8753 &mac_profile, extack); 8754 if (err) 8755 return err; 8756 8757 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8758 rif->mac_profile_id = mac_profile; 8759 return 0; 8760 } 8761 8762 static int 8763 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8764 struct net_device *l3_dev, 8765 struct netlink_ext_ack *extack) 8766 { 8767 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8768 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8769 struct mlxsw_sp_rif_params params = { 8770 .dev = l3_dev, 8771 }; 8772 u16 vid = mlxsw_sp_port_vlan->vid; 8773 struct mlxsw_sp_rif *rif; 8774 struct mlxsw_sp_fid *fid; 8775 int err; 8776 8777 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8778 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8779 if (IS_ERR(rif)) 8780 return PTR_ERR(rif); 8781 8782 /* FID was already created, just take a reference */ 8783 fid = rif->ops->fid_get(rif, extack); 8784 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8785 if (err) 8786 goto err_fid_port_vid_map; 8787 8788 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8789 if (err) 8790 goto err_port_vid_learning_set; 8791 8792 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8793 BR_STATE_FORWARDING); 8794 if (err) 8795 goto err_port_vid_stp_set; 8796 8797 mlxsw_sp_port_vlan->fid = fid; 8798 8799 return 0; 8800 8801 err_port_vid_stp_set: 8802 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8803 err_port_vid_learning_set: 8804 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8805 err_fid_port_vid_map: 8806 mlxsw_sp_fid_put(fid); 8807 mlxsw_sp_rif_subport_put(rif); 8808 return err; 8809 } 8810 8811 static void 8812 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8813 { 8814 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8815 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8816 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8817 u16 vid = mlxsw_sp_port_vlan->vid; 8818 8819 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8820 return; 8821 8822 mlxsw_sp_port_vlan->fid = NULL; 8823 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8824 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8825 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8826 mlxsw_sp_fid_put(fid); 8827 mlxsw_sp_rif_subport_put(rif); 8828 } 8829 8830 int 8831 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8832 struct net_device *l3_dev, 8833 struct netlink_ext_ack *extack) 8834 { 8835 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8836 struct mlxsw_sp_rif *rif; 8837 int err = 0; 8838 8839 mutex_lock(&mlxsw_sp->router->lock); 8840 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8841 if (!rif) 8842 goto out; 8843 8844 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8845 extack); 8846 out: 8847 mutex_unlock(&mlxsw_sp->router->lock); 8848 return err; 8849 } 8850 8851 void 8852 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8853 { 8854 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8855 8856 mutex_lock(&mlxsw_sp->router->lock); 8857 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8858 mutex_unlock(&mlxsw_sp->router->lock); 8859 } 8860 8861 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8862 struct net_device *port_dev, 8863 unsigned long event, u16 vid, 8864 struct netlink_ext_ack *extack) 8865 { 8866 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8867 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8868 8869 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8870 if (WARN_ON(!mlxsw_sp_port_vlan)) 8871 return -EINVAL; 8872 8873 switch (event) { 8874 case NETDEV_UP: 8875 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8876 l3_dev, extack); 8877 case NETDEV_DOWN: 8878 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8879 break; 8880 } 8881 8882 return 0; 8883 } 8884 8885 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8886 unsigned long event, 8887 struct netlink_ext_ack *extack) 8888 { 8889 if (netif_is_bridge_port(port_dev) || 8890 netif_is_lag_port(port_dev) || 8891 netif_is_ovs_port(port_dev)) 8892 return 0; 8893 8894 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8895 MLXSW_SP_DEFAULT_VID, extack); 8896 } 8897 8898 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8899 struct net_device *lag_dev, 8900 unsigned long event, u16 vid, 8901 struct netlink_ext_ack *extack) 8902 { 8903 struct net_device *port_dev; 8904 struct list_head *iter; 8905 int err; 8906 8907 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8908 if (mlxsw_sp_port_dev_check(port_dev)) { 8909 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8910 port_dev, 8911 event, vid, 8912 extack); 8913 if (err) 8914 return err; 8915 } 8916 } 8917 8918 return 0; 8919 } 8920 8921 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8922 unsigned long event, 8923 struct netlink_ext_ack *extack) 8924 { 8925 if (netif_is_bridge_port(lag_dev)) 8926 return 0; 8927 8928 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8929 MLXSW_SP_DEFAULT_VID, extack); 8930 } 8931 8932 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8933 struct net_device *l3_dev, 8934 unsigned long event, 8935 struct netlink_ext_ack *extack) 8936 { 8937 struct mlxsw_sp_rif_params params = { 8938 .dev = l3_dev, 8939 }; 8940 struct mlxsw_sp_rif *rif; 8941 8942 switch (event) { 8943 case NETDEV_UP: 8944 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8945 u16 proto; 8946 8947 br_vlan_get_proto(l3_dev, &proto); 8948 if (proto == ETH_P_8021AD) { 8949 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8950 return -EOPNOTSUPP; 8951 } 8952 } 8953 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8954 if (IS_ERR(rif)) 8955 return PTR_ERR(rif); 8956 break; 8957 case NETDEV_DOWN: 8958 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8959 mlxsw_sp_rif_destroy(rif); 8960 break; 8961 } 8962 8963 return 0; 8964 } 8965 8966 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8967 struct net_device *vlan_dev, 8968 unsigned long event, 8969 struct netlink_ext_ack *extack) 8970 { 8971 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8972 u16 vid = vlan_dev_vlan_id(vlan_dev); 8973 8974 if (netif_is_bridge_port(vlan_dev)) 8975 return 0; 8976 8977 if (mlxsw_sp_port_dev_check(real_dev)) 8978 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8979 event, vid, extack); 8980 else if (netif_is_lag_master(real_dev)) 8981 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8982 vid, extack); 8983 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8984 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8985 extack); 8986 8987 return 0; 8988 } 8989 8990 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8991 { 8992 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8993 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8994 8995 return ether_addr_equal_masked(mac, vrrp4, mask); 8996 } 8997 8998 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8999 { 9000 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 9001 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 9002 9003 return ether_addr_equal_masked(mac, vrrp6, mask); 9004 } 9005 9006 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9007 const u8 *mac, bool adding) 9008 { 9009 char ritr_pl[MLXSW_REG_RITR_LEN]; 9010 u8 vrrp_id = adding ? mac[5] : 0; 9011 int err; 9012 9013 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 9014 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 9015 return 0; 9016 9017 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9018 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9019 if (err) 9020 return err; 9021 9022 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 9023 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 9024 else 9025 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 9026 9027 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9028 } 9029 9030 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 9031 const struct net_device *macvlan_dev, 9032 struct netlink_ext_ack *extack) 9033 { 9034 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9035 struct mlxsw_sp_rif *rif; 9036 int err; 9037 9038 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9039 if (!rif) { 9040 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 9041 return -EOPNOTSUPP; 9042 } 9043 9044 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9045 mlxsw_sp_fid_index(rif->fid), true); 9046 if (err) 9047 return err; 9048 9049 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 9050 macvlan_dev->dev_addr, true); 9051 if (err) 9052 goto err_rif_vrrp_add; 9053 9054 /* Make sure the bridge driver does not have this MAC pointing at 9055 * some other port. 9056 */ 9057 if (rif->ops->fdb_del) 9058 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 9059 9060 return 0; 9061 9062 err_rif_vrrp_add: 9063 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9064 mlxsw_sp_fid_index(rif->fid), false); 9065 return err; 9066 } 9067 9068 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9069 const struct net_device *macvlan_dev) 9070 { 9071 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 9072 struct mlxsw_sp_rif *rif; 9073 9074 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 9075 /* If we do not have a RIF, then we already took care of 9076 * removing the macvlan's MAC during RIF deletion. 9077 */ 9078 if (!rif) 9079 return; 9080 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 9081 false); 9082 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 9083 mlxsw_sp_fid_index(rif->fid), false); 9084 } 9085 9086 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 9087 const struct net_device *macvlan_dev) 9088 { 9089 mutex_lock(&mlxsw_sp->router->lock); 9090 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9091 mutex_unlock(&mlxsw_sp->router->lock); 9092 } 9093 9094 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 9095 struct net_device *macvlan_dev, 9096 unsigned long event, 9097 struct netlink_ext_ack *extack) 9098 { 9099 switch (event) { 9100 case NETDEV_UP: 9101 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 9102 case NETDEV_DOWN: 9103 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 9104 break; 9105 } 9106 9107 return 0; 9108 } 9109 9110 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9111 struct net_device *dev, 9112 unsigned long event, 9113 struct netlink_ext_ack *extack) 9114 { 9115 if (mlxsw_sp_port_dev_check(dev)) 9116 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 9117 else if (netif_is_lag_master(dev)) 9118 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 9119 else if (netif_is_bridge_master(dev)) 9120 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 9121 extack); 9122 else if (is_vlan_dev(dev)) 9123 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9124 extack); 9125 else if (netif_is_macvlan(dev)) 9126 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9127 extack); 9128 else 9129 return 0; 9130 } 9131 9132 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9133 unsigned long event, void *ptr) 9134 { 9135 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9136 struct net_device *dev = ifa->ifa_dev->dev; 9137 struct mlxsw_sp_router *router; 9138 struct mlxsw_sp_rif *rif; 9139 int err = 0; 9140 9141 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9142 if (event == NETDEV_UP) 9143 return NOTIFY_DONE; 9144 9145 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9146 mutex_lock(&router->lock); 9147 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9148 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9149 goto out; 9150 9151 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 9152 out: 9153 mutex_unlock(&router->lock); 9154 return notifier_from_errno(err); 9155 } 9156 9157 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9158 unsigned long event, void *ptr) 9159 { 9160 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9161 struct net_device *dev = ivi->ivi_dev->dev; 9162 struct mlxsw_sp *mlxsw_sp; 9163 struct mlxsw_sp_rif *rif; 9164 int err = 0; 9165 9166 mlxsw_sp = mlxsw_sp_lower_get(dev); 9167 if (!mlxsw_sp) 9168 return NOTIFY_DONE; 9169 9170 mutex_lock(&mlxsw_sp->router->lock); 9171 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9172 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9173 goto out; 9174 9175 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 9176 out: 9177 mutex_unlock(&mlxsw_sp->router->lock); 9178 return notifier_from_errno(err); 9179 } 9180 9181 struct mlxsw_sp_inet6addr_event_work { 9182 struct work_struct work; 9183 struct mlxsw_sp *mlxsw_sp; 9184 struct net_device *dev; 9185 unsigned long event; 9186 }; 9187 9188 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9189 { 9190 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9191 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9192 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9193 struct net_device *dev = inet6addr_work->dev; 9194 unsigned long event = inet6addr_work->event; 9195 struct mlxsw_sp_rif *rif; 9196 9197 rtnl_lock(); 9198 mutex_lock(&mlxsw_sp->router->lock); 9199 9200 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9201 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9202 goto out; 9203 9204 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 9205 out: 9206 mutex_unlock(&mlxsw_sp->router->lock); 9207 rtnl_unlock(); 9208 dev_put(dev); 9209 kfree(inet6addr_work); 9210 } 9211 9212 /* Called with rcu_read_lock() */ 9213 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9214 unsigned long event, void *ptr) 9215 { 9216 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9217 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9218 struct net_device *dev = if6->idev->dev; 9219 struct mlxsw_sp_router *router; 9220 9221 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9222 if (event == NETDEV_UP) 9223 return NOTIFY_DONE; 9224 9225 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 9226 if (!inet6addr_work) 9227 return NOTIFY_BAD; 9228 9229 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9230 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9231 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9232 inet6addr_work->dev = dev; 9233 inet6addr_work->event = event; 9234 dev_hold(dev); 9235 mlxsw_core_schedule_work(&inet6addr_work->work); 9236 9237 return NOTIFY_DONE; 9238 } 9239 9240 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9241 unsigned long event, void *ptr) 9242 { 9243 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9244 struct net_device *dev = i6vi->i6vi_dev->dev; 9245 struct mlxsw_sp *mlxsw_sp; 9246 struct mlxsw_sp_rif *rif; 9247 int err = 0; 9248 9249 mlxsw_sp = mlxsw_sp_lower_get(dev); 9250 if (!mlxsw_sp) 9251 return NOTIFY_DONE; 9252 9253 mutex_lock(&mlxsw_sp->router->lock); 9254 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9255 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9256 goto out; 9257 9258 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 9259 out: 9260 mutex_unlock(&mlxsw_sp->router->lock); 9261 return notifier_from_errno(err); 9262 } 9263 9264 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9265 const char *mac, int mtu, u8 mac_profile) 9266 { 9267 char ritr_pl[MLXSW_REG_RITR_LEN]; 9268 int err; 9269 9270 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9271 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9272 if (err) 9273 return err; 9274 9275 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9276 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9277 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9278 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9279 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9280 } 9281 9282 static int 9283 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9284 struct mlxsw_sp_rif *rif, 9285 struct netlink_ext_ack *extack) 9286 { 9287 struct net_device *dev = rif->dev; 9288 u8 old_mac_profile; 9289 u16 fid_index; 9290 int err; 9291 9292 fid_index = mlxsw_sp_fid_index(rif->fid); 9293 9294 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9295 if (err) 9296 return err; 9297 9298 old_mac_profile = rif->mac_profile_id; 9299 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9300 extack); 9301 if (err) 9302 goto err_rif_mac_profile_replace; 9303 9304 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9305 dev->mtu, rif->mac_profile_id); 9306 if (err) 9307 goto err_rif_edit; 9308 9309 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9310 if (err) 9311 goto err_rif_fdb_op; 9312 9313 if (rif->mtu != dev->mtu) { 9314 struct mlxsw_sp_vr *vr; 9315 int i; 9316 9317 /* The RIF is relevant only to its mr_table instance, as unlike 9318 * unicast routing, in multicast routing a RIF cannot be shared 9319 * between several multicast routing tables. 9320 */ 9321 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9322 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9323 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9324 rif, dev->mtu); 9325 } 9326 9327 ether_addr_copy(rif->addr, dev->dev_addr); 9328 rif->mtu = dev->mtu; 9329 9330 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9331 9332 return 0; 9333 9334 err_rif_fdb_op: 9335 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9336 old_mac_profile); 9337 err_rif_edit: 9338 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9339 err_rif_mac_profile_replace: 9340 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9341 return err; 9342 } 9343 9344 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9345 struct netdev_notifier_pre_changeaddr_info *info) 9346 { 9347 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9348 struct mlxsw_sp_rif_mac_profile *profile; 9349 struct netlink_ext_ack *extack; 9350 u8 max_rif_mac_profiles; 9351 u64 occ; 9352 9353 extack = netdev_notifier_info_to_extack(&info->info); 9354 9355 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9356 if (profile) 9357 return 0; 9358 9359 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9360 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9361 if (occ < max_rif_mac_profiles) 9362 return 0; 9363 9364 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9365 return 0; 9366 9367 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9368 return -ENOBUFS; 9369 } 9370 9371 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9372 { 9373 switch (event) { 9374 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9375 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9376 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9377 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9378 return true; 9379 } 9380 9381 return false; 9382 } 9383 9384 static int 9385 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9386 unsigned long event, 9387 struct netdev_notifier_offload_xstats_info *info) 9388 { 9389 switch (info->type) { 9390 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9391 break; 9392 default: 9393 return 0; 9394 } 9395 9396 switch (event) { 9397 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9398 return mlxsw_sp_router_port_l3_stats_enable(rif); 9399 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9400 mlxsw_sp_router_port_l3_stats_disable(rif); 9401 return 0; 9402 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9403 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9404 return 0; 9405 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9406 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9407 } 9408 9409 WARN_ON_ONCE(1); 9410 return 0; 9411 } 9412 9413 static int 9414 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9415 struct net_device *dev, 9416 unsigned long event, 9417 struct netdev_notifier_offload_xstats_info *info) 9418 { 9419 struct mlxsw_sp_rif *rif; 9420 9421 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9422 if (!rif) 9423 return 0; 9424 9425 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9426 } 9427 9428 static bool mlxsw_sp_is_router_event(unsigned long event) 9429 { 9430 switch (event) { 9431 case NETDEV_PRE_CHANGEADDR: 9432 case NETDEV_CHANGEADDR: 9433 case NETDEV_CHANGEMTU: 9434 return true; 9435 default: 9436 return false; 9437 } 9438 } 9439 9440 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9441 unsigned long event, void *ptr) 9442 { 9443 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9444 struct mlxsw_sp *mlxsw_sp; 9445 struct mlxsw_sp_rif *rif; 9446 9447 mlxsw_sp = mlxsw_sp_lower_get(dev); 9448 if (!mlxsw_sp) 9449 return 0; 9450 9451 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9452 if (!rif) 9453 return 0; 9454 9455 switch (event) { 9456 case NETDEV_CHANGEMTU: 9457 case NETDEV_CHANGEADDR: 9458 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9459 case NETDEV_PRE_CHANGEADDR: 9460 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9461 default: 9462 WARN_ON_ONCE(1); 9463 break; 9464 } 9465 9466 return 0; 9467 } 9468 9469 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9470 struct net_device *l3_dev, 9471 struct netlink_ext_ack *extack) 9472 { 9473 struct mlxsw_sp_rif *rif; 9474 9475 /* If netdev is already associated with a RIF, then we need to 9476 * destroy it and create a new one with the new virtual router ID. 9477 */ 9478 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9479 if (rif) 9480 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 9481 extack); 9482 9483 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 9484 } 9485 9486 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9487 struct net_device *l3_dev) 9488 { 9489 struct mlxsw_sp_rif *rif; 9490 9491 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9492 if (!rif) 9493 return; 9494 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 9495 } 9496 9497 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9498 { 9499 struct netdev_notifier_changeupper_info *info = ptr; 9500 9501 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9502 return false; 9503 return netif_is_l3_master(info->upper_dev); 9504 } 9505 9506 static int 9507 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9508 struct netdev_notifier_changeupper_info *info) 9509 { 9510 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9511 int err = 0; 9512 9513 /* We do not create a RIF for a macvlan, but only use it to 9514 * direct more MAC addresses to the router. 9515 */ 9516 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9517 return 0; 9518 9519 switch (event) { 9520 case NETDEV_PRECHANGEUPPER: 9521 break; 9522 case NETDEV_CHANGEUPPER: 9523 if (info->linking) { 9524 struct netlink_ext_ack *extack; 9525 9526 extack = netdev_notifier_info_to_extack(&info->info); 9527 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9528 } else { 9529 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9530 } 9531 break; 9532 } 9533 9534 return err; 9535 } 9536 9537 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 9538 unsigned long event, void *ptr) 9539 { 9540 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 9541 struct mlxsw_sp_router *router; 9542 struct mlxsw_sp *mlxsw_sp; 9543 int err = 0; 9544 9545 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 9546 mlxsw_sp = router->mlxsw_sp; 9547 9548 mutex_lock(&mlxsw_sp->router->lock); 9549 9550 if (mlxsw_sp_is_offload_xstats_event(event)) 9551 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 9552 event, ptr); 9553 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 9554 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 9555 event, ptr); 9556 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 9557 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 9558 event, ptr); 9559 else if (mlxsw_sp_is_router_event(event)) 9560 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 9561 else if (mlxsw_sp_is_vrf_event(event, ptr)) 9562 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 9563 9564 mutex_unlock(&mlxsw_sp->router->lock); 9565 9566 return notifier_from_errno(err); 9567 } 9568 9569 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 9570 struct netdev_nested_priv *priv) 9571 { 9572 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 9573 9574 if (!netif_is_macvlan(dev)) 9575 return 0; 9576 9577 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9578 mlxsw_sp_fid_index(rif->fid), false); 9579 } 9580 9581 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 9582 { 9583 struct netdev_nested_priv priv = { 9584 .data = (void *)rif, 9585 }; 9586 9587 if (!netif_is_macvlan_port(rif->dev)) 9588 return 0; 9589 9590 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 9591 return netdev_walk_all_upper_dev_rcu(rif->dev, 9592 __mlxsw_sp_rif_macvlan_flush, &priv); 9593 } 9594 9595 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 9596 const struct mlxsw_sp_rif_params *params) 9597 { 9598 struct mlxsw_sp_rif_subport *rif_subport; 9599 9600 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9601 refcount_set(&rif_subport->ref_count, 1); 9602 rif_subport->vid = params->vid; 9603 rif_subport->lag = params->lag; 9604 if (params->lag) 9605 rif_subport->lag_id = params->lag_id; 9606 else 9607 rif_subport->system_port = params->system_port; 9608 } 9609 9610 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 9611 { 9612 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9613 struct mlxsw_sp_rif_subport *rif_subport; 9614 char ritr_pl[MLXSW_REG_RITR_LEN]; 9615 9616 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9617 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 9618 rif->rif_index, rif->vr_id, rif->dev->mtu); 9619 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9620 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9621 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 9622 rif_subport->lag ? rif_subport->lag_id : 9623 rif_subport->system_port, 9624 rif_subport->vid); 9625 9626 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9627 } 9628 9629 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 9630 struct netlink_ext_ack *extack) 9631 { 9632 u8 mac_profile; 9633 int err; 9634 9635 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 9636 &mac_profile, extack); 9637 if (err) 9638 return err; 9639 rif->mac_profile_id = mac_profile; 9640 9641 err = mlxsw_sp_rif_subport_op(rif, true); 9642 if (err) 9643 goto err_rif_subport_op; 9644 9645 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9646 mlxsw_sp_fid_index(rif->fid), true); 9647 if (err) 9648 goto err_rif_fdb_op; 9649 9650 mlxsw_sp_fid_rif_set(rif->fid, rif); 9651 return 0; 9652 9653 err_rif_fdb_op: 9654 mlxsw_sp_rif_subport_op(rif, false); 9655 err_rif_subport_op: 9656 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 9657 return err; 9658 } 9659 9660 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 9661 { 9662 struct mlxsw_sp_fid *fid = rif->fid; 9663 9664 mlxsw_sp_fid_rif_set(fid, NULL); 9665 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9666 mlxsw_sp_fid_index(fid), false); 9667 mlxsw_sp_rif_macvlan_flush(rif); 9668 mlxsw_sp_rif_subport_op(rif, false); 9669 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9670 } 9671 9672 static struct mlxsw_sp_fid * 9673 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 9674 struct netlink_ext_ack *extack) 9675 { 9676 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 9677 } 9678 9679 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 9680 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 9681 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 9682 .setup = mlxsw_sp_rif_subport_setup, 9683 .configure = mlxsw_sp_rif_subport_configure, 9684 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 9685 .fid_get = mlxsw_sp_rif_subport_fid_get, 9686 }; 9687 9688 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, 9689 enum mlxsw_reg_ritr_if_type type, 9690 u16 vid_fid, bool enable) 9691 { 9692 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9693 char ritr_pl[MLXSW_REG_RITR_LEN]; 9694 9695 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9696 rif->dev->mtu); 9697 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9698 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9699 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid); 9700 9701 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9702 } 9703 9704 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9705 { 9706 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9707 } 9708 9709 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 9710 struct netlink_ext_ack *extack) 9711 { 9712 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9713 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9714 u8 mac_profile; 9715 int err; 9716 9717 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 9718 &mac_profile, extack); 9719 if (err) 9720 return err; 9721 rif->mac_profile_id = mac_profile; 9722 9723 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, 9724 true); 9725 if (err) 9726 goto err_rif_vlan_fid_op; 9727 9728 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9729 mlxsw_sp_router_port(mlxsw_sp), true); 9730 if (err) 9731 goto err_fid_mc_flood_set; 9732 9733 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9734 mlxsw_sp_router_port(mlxsw_sp), true); 9735 if (err) 9736 goto err_fid_bc_flood_set; 9737 9738 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9739 mlxsw_sp_fid_index(rif->fid), true); 9740 if (err) 9741 goto err_rif_fdb_op; 9742 9743 mlxsw_sp_fid_rif_set(rif->fid, rif); 9744 return 0; 9745 9746 err_rif_fdb_op: 9747 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9748 mlxsw_sp_router_port(mlxsw_sp), false); 9749 err_fid_bc_flood_set: 9750 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9751 mlxsw_sp_router_port(mlxsw_sp), false); 9752 err_fid_mc_flood_set: 9753 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9754 err_rif_vlan_fid_op: 9755 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 9756 return err; 9757 } 9758 9759 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9760 { 9761 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9762 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9763 struct mlxsw_sp_fid *fid = rif->fid; 9764 9765 mlxsw_sp_fid_rif_set(fid, NULL); 9766 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9767 mlxsw_sp_fid_index(fid), false); 9768 mlxsw_sp_rif_macvlan_flush(rif); 9769 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9770 mlxsw_sp_router_port(mlxsw_sp), false); 9771 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9772 mlxsw_sp_router_port(mlxsw_sp), false); 9773 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false); 9774 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9775 } 9776 9777 static struct mlxsw_sp_fid * 9778 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9779 struct netlink_ext_ack *extack) 9780 { 9781 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 9782 } 9783 9784 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9785 { 9786 struct switchdev_notifier_fdb_info info = {}; 9787 struct net_device *dev; 9788 9789 dev = br_fdb_find_port(rif->dev, mac, 0); 9790 if (!dev) 9791 return; 9792 9793 info.addr = mac; 9794 info.vid = 0; 9795 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9796 NULL); 9797 } 9798 9799 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9800 .type = MLXSW_SP_RIF_TYPE_FID, 9801 .rif_size = sizeof(struct mlxsw_sp_rif), 9802 .configure = mlxsw_sp_rif_fid_configure, 9803 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9804 .fid_get = mlxsw_sp_rif_fid_fid_get, 9805 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9806 }; 9807 9808 static struct mlxsw_sp_fid * 9809 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9810 struct netlink_ext_ack *extack) 9811 { 9812 struct net_device *br_dev; 9813 u16 vid; 9814 int err; 9815 9816 if (is_vlan_dev(rif->dev)) { 9817 vid = vlan_dev_vlan_id(rif->dev); 9818 br_dev = vlan_dev_real_dev(rif->dev); 9819 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9820 return ERR_PTR(-EINVAL); 9821 } else { 9822 err = br_vlan_get_pvid(rif->dev, &vid); 9823 if (err < 0 || !vid) { 9824 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9825 return ERR_PTR(-EINVAL); 9826 } 9827 } 9828 9829 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9830 } 9831 9832 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9833 { 9834 struct switchdev_notifier_fdb_info info = {}; 9835 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9836 struct net_device *br_dev; 9837 struct net_device *dev; 9838 9839 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 9840 dev = br_fdb_find_port(br_dev, mac, vid); 9841 if (!dev) 9842 return; 9843 9844 info.addr = mac; 9845 info.vid = vid; 9846 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9847 NULL); 9848 } 9849 9850 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 9851 .type = MLXSW_SP_RIF_TYPE_VLAN, 9852 .rif_size = sizeof(struct mlxsw_sp_rif), 9853 .configure = mlxsw_sp_rif_fid_configure, 9854 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9855 .fid_get = mlxsw_sp_rif_vlan_fid_get, 9856 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 9857 }; 9858 9859 static struct mlxsw_sp_rif_ipip_lb * 9860 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 9861 { 9862 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 9863 } 9864 9865 static void 9866 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 9867 const struct mlxsw_sp_rif_params *params) 9868 { 9869 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 9870 struct mlxsw_sp_rif_ipip_lb *rif_lb; 9871 9872 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 9873 common); 9874 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 9875 rif_lb->lb_config = params_lb->lb_config; 9876 } 9877 9878 static int 9879 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 9880 struct netlink_ext_ack *extack) 9881 { 9882 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9883 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9884 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9885 struct mlxsw_sp_vr *ul_vr; 9886 int err; 9887 9888 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 9889 if (IS_ERR(ul_vr)) 9890 return PTR_ERR(ul_vr); 9891 9892 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 9893 if (err) 9894 goto err_loopback_op; 9895 9896 lb_rif->ul_vr_id = ul_vr->id; 9897 lb_rif->ul_rif_id = 0; 9898 ++ul_vr->rif_count; 9899 return 0; 9900 9901 err_loopback_op: 9902 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9903 return err; 9904 } 9905 9906 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9907 { 9908 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9909 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9910 struct mlxsw_sp_vr *ul_vr; 9911 9912 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 9913 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 9914 9915 --ul_vr->rif_count; 9916 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9917 } 9918 9919 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 9920 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9921 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9922 .setup = mlxsw_sp_rif_ipip_lb_setup, 9923 .configure = mlxsw_sp1_rif_ipip_lb_configure, 9924 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 9925 }; 9926 9927 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 9928 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9929 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 9930 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9931 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 9932 }; 9933 9934 static int 9935 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 9936 { 9937 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9938 char ritr_pl[MLXSW_REG_RITR_LEN]; 9939 9940 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 9941 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 9942 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 9943 MLXSW_REG_RITR_LOOPBACK_GENERIC); 9944 9945 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9946 } 9947 9948 static struct mlxsw_sp_rif * 9949 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 9950 struct netlink_ext_ack *extack) 9951 { 9952 struct mlxsw_sp_rif *ul_rif; 9953 u16 rif_index; 9954 int err; 9955 9956 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 9957 if (err) { 9958 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 9959 return ERR_PTR(err); 9960 } 9961 9962 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 9963 if (!ul_rif) 9964 return ERR_PTR(-ENOMEM); 9965 9966 mlxsw_sp->router->rifs[rif_index] = ul_rif; 9967 ul_rif->mlxsw_sp = mlxsw_sp; 9968 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 9969 if (err) 9970 goto ul_rif_op_err; 9971 9972 return ul_rif; 9973 9974 ul_rif_op_err: 9975 mlxsw_sp->router->rifs[rif_index] = NULL; 9976 kfree(ul_rif); 9977 return ERR_PTR(err); 9978 } 9979 9980 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9981 { 9982 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9983 9984 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9985 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9986 kfree(ul_rif); 9987 } 9988 9989 static struct mlxsw_sp_rif * 9990 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 9991 struct netlink_ext_ack *extack) 9992 { 9993 struct mlxsw_sp_vr *vr; 9994 int err; 9995 9996 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 9997 if (IS_ERR(vr)) 9998 return ERR_CAST(vr); 9999 10000 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10001 return vr->ul_rif; 10002 10003 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 10004 if (IS_ERR(vr->ul_rif)) { 10005 err = PTR_ERR(vr->ul_rif); 10006 goto err_ul_rif_create; 10007 } 10008 10009 vr->rif_count++; 10010 refcount_set(&vr->ul_rif_refcnt, 1); 10011 10012 return vr->ul_rif; 10013 10014 err_ul_rif_create: 10015 mlxsw_sp_vr_put(mlxsw_sp, vr); 10016 return ERR_PTR(err); 10017 } 10018 10019 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10020 { 10021 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10022 struct mlxsw_sp_vr *vr; 10023 10024 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10025 10026 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10027 return; 10028 10029 vr->rif_count--; 10030 mlxsw_sp_ul_rif_destroy(ul_rif); 10031 mlxsw_sp_vr_put(mlxsw_sp, vr); 10032 } 10033 10034 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10035 u16 *ul_rif_index) 10036 { 10037 struct mlxsw_sp_rif *ul_rif; 10038 int err = 0; 10039 10040 mutex_lock(&mlxsw_sp->router->lock); 10041 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 10042 if (IS_ERR(ul_rif)) { 10043 err = PTR_ERR(ul_rif); 10044 goto out; 10045 } 10046 *ul_rif_index = ul_rif->rif_index; 10047 out: 10048 mutex_unlock(&mlxsw_sp->router->lock); 10049 return err; 10050 } 10051 10052 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10053 { 10054 struct mlxsw_sp_rif *ul_rif; 10055 10056 mutex_lock(&mlxsw_sp->router->lock); 10057 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10058 if (WARN_ON(!ul_rif)) 10059 goto out; 10060 10061 mlxsw_sp_ul_rif_put(ul_rif); 10062 out: 10063 mutex_unlock(&mlxsw_sp->router->lock); 10064 } 10065 10066 static int 10067 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10068 struct netlink_ext_ack *extack) 10069 { 10070 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10071 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 10072 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10073 struct mlxsw_sp_rif *ul_rif; 10074 int err; 10075 10076 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 10077 if (IS_ERR(ul_rif)) 10078 return PTR_ERR(ul_rif); 10079 10080 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10081 if (err) 10082 goto err_loopback_op; 10083 10084 lb_rif->ul_vr_id = 0; 10085 lb_rif->ul_rif_id = ul_rif->rif_index; 10086 10087 return 0; 10088 10089 err_loopback_op: 10090 mlxsw_sp_ul_rif_put(ul_rif); 10091 return err; 10092 } 10093 10094 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10095 { 10096 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10097 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10098 struct mlxsw_sp_rif *ul_rif; 10099 10100 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 10101 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 10102 mlxsw_sp_ul_rif_put(ul_rif); 10103 } 10104 10105 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 10106 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10107 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10108 .setup = mlxsw_sp_rif_ipip_lb_setup, 10109 .configure = mlxsw_sp2_rif_ipip_lb_configure, 10110 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 10111 }; 10112 10113 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 10114 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10115 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_emu_ops, 10116 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10117 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 10118 }; 10119 10120 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 10121 { 10122 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10123 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10124 struct mlxsw_core *core = mlxsw_sp->core; 10125 10126 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 10127 return -EIO; 10128 mlxsw_sp->router->max_rif_mac_profile = 10129 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 10130 10131 mlxsw_sp->router->rifs = kcalloc(max_rifs, 10132 sizeof(struct mlxsw_sp_rif *), 10133 GFP_KERNEL); 10134 if (!mlxsw_sp->router->rifs) 10135 return -ENOMEM; 10136 10137 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 10138 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 10139 devlink_resource_occ_get_register(devlink, 10140 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 10141 mlxsw_sp_rif_mac_profiles_occ_get, 10142 mlxsw_sp); 10143 10144 return 0; 10145 } 10146 10147 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 10148 { 10149 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10150 int i; 10151 10152 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 10153 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 10154 10155 devlink_resource_occ_get_unregister(devlink, 10156 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 10157 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 10158 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 10159 kfree(mlxsw_sp->router->rifs); 10160 } 10161 10162 static int 10163 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 10164 { 10165 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 10166 10167 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 10168 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 10169 } 10170 10171 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 10172 { 10173 int err; 10174 10175 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 10176 10177 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 10178 if (err) 10179 return err; 10180 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 10181 if (err) 10182 return err; 10183 10184 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 10185 } 10186 10187 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 10188 { 10189 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 10190 return mlxsw_sp_ipips_init(mlxsw_sp); 10191 } 10192 10193 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 10194 { 10195 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 10196 return mlxsw_sp_ipips_init(mlxsw_sp); 10197 } 10198 10199 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 10200 { 10201 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 10202 } 10203 10204 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 10205 { 10206 struct mlxsw_sp_router *router; 10207 10208 /* Flush pending FIB notifications and then flush the device's 10209 * table before requesting another dump. The FIB notification 10210 * block is unregistered, so no need to take RTNL. 10211 */ 10212 mlxsw_core_flush_owq(); 10213 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 10214 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 10215 } 10216 10217 #ifdef CONFIG_IP_ROUTE_MULTIPATH 10218 struct mlxsw_sp_mp_hash_config { 10219 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 10220 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 10221 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 10222 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 10223 bool inc_parsing_depth; 10224 }; 10225 10226 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 10227 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 10228 10229 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 10230 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 10231 10232 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 10233 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 10234 10235 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 10236 { 10237 unsigned long *inner_headers = config->inner_headers; 10238 unsigned long *inner_fields = config->inner_fields; 10239 10240 /* IPv4 inner */ 10241 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10242 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10243 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10244 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10245 /* IPv6 inner */ 10246 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10247 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10248 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10249 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10250 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10251 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10252 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10253 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10254 } 10255 10256 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10257 { 10258 unsigned long *headers = config->headers; 10259 unsigned long *fields = config->fields; 10260 10261 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10262 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10263 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10264 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10265 } 10266 10267 static void 10268 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 10269 u32 hash_fields) 10270 { 10271 unsigned long *inner_headers = config->inner_headers; 10272 unsigned long *inner_fields = config->inner_fields; 10273 10274 /* IPv4 Inner */ 10275 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10276 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10277 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 10278 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10279 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 10280 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10281 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10282 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 10283 /* IPv6 inner */ 10284 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10285 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10286 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 10287 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10288 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10289 } 10290 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 10291 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10292 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10293 } 10294 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10295 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10296 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 10297 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10298 /* L4 inner */ 10299 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 10300 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 10301 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 10302 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 10303 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 10304 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 10305 } 10306 10307 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 10308 struct mlxsw_sp_mp_hash_config *config) 10309 { 10310 struct net *net = mlxsw_sp_net(mlxsw_sp); 10311 unsigned long *headers = config->headers; 10312 unsigned long *fields = config->fields; 10313 u32 hash_fields; 10314 10315 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 10316 case 0: 10317 mlxsw_sp_mp4_hash_outer_addr(config); 10318 break; 10319 case 1: 10320 mlxsw_sp_mp4_hash_outer_addr(config); 10321 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10322 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10323 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10324 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10325 break; 10326 case 2: 10327 /* Outer */ 10328 mlxsw_sp_mp4_hash_outer_addr(config); 10329 /* Inner */ 10330 mlxsw_sp_mp_hash_inner_l3(config); 10331 break; 10332 case 3: 10333 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; 10334 /* Outer */ 10335 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10336 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10337 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10338 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 10339 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10340 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 10341 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10342 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10343 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10344 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10345 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10346 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10347 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10348 /* Inner */ 10349 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10350 break; 10351 } 10352 } 10353 10354 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10355 { 10356 unsigned long *headers = config->headers; 10357 unsigned long *fields = config->fields; 10358 10359 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10360 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10361 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10362 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10363 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10364 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10365 } 10366 10367 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 10368 struct mlxsw_sp_mp_hash_config *config) 10369 { 10370 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 10371 unsigned long *headers = config->headers; 10372 unsigned long *fields = config->fields; 10373 10374 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 10375 case 0: 10376 mlxsw_sp_mp6_hash_outer_addr(config); 10377 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10378 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10379 break; 10380 case 1: 10381 mlxsw_sp_mp6_hash_outer_addr(config); 10382 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10383 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10384 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10385 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10386 break; 10387 case 2: 10388 /* Outer */ 10389 mlxsw_sp_mp6_hash_outer_addr(config); 10390 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10391 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10392 /* Inner */ 10393 mlxsw_sp_mp_hash_inner_l3(config); 10394 config->inc_parsing_depth = true; 10395 break; 10396 case 3: 10397 /* Outer */ 10398 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10399 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10400 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10401 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 10402 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10403 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10404 } 10405 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 10406 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10407 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10408 } 10409 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10410 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10411 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 10412 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10413 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10414 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10415 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10416 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10417 /* Inner */ 10418 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10419 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 10420 config->inc_parsing_depth = true; 10421 break; 10422 } 10423 } 10424 10425 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 10426 bool old_inc_parsing_depth, 10427 bool new_inc_parsing_depth) 10428 { 10429 int err; 10430 10431 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 10432 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 10433 if (err) 10434 return err; 10435 mlxsw_sp->router->inc_parsing_depth = true; 10436 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 10437 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 10438 mlxsw_sp->router->inc_parsing_depth = false; 10439 } 10440 10441 return 0; 10442 } 10443 10444 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10445 { 10446 bool old_inc_parsing_depth, new_inc_parsing_depth; 10447 struct mlxsw_sp_mp_hash_config config = {}; 10448 char recr2_pl[MLXSW_REG_RECR2_LEN]; 10449 unsigned long bit; 10450 u32 seed; 10451 int err; 10452 10453 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 10454 mlxsw_reg_recr2_pack(recr2_pl, seed); 10455 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 10456 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 10457 10458 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 10459 new_inc_parsing_depth = config.inc_parsing_depth; 10460 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 10461 old_inc_parsing_depth, 10462 new_inc_parsing_depth); 10463 if (err) 10464 return err; 10465 10466 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 10467 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 10468 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 10469 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 10470 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 10471 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 10472 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 10473 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 10474 10475 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 10476 if (err) 10477 goto err_reg_write; 10478 10479 return 0; 10480 10481 err_reg_write: 10482 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 10483 old_inc_parsing_depth); 10484 return err; 10485 } 10486 #else 10487 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10488 { 10489 return 0; 10490 } 10491 #endif 10492 10493 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 10494 { 10495 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 10496 unsigned int i; 10497 10498 MLXSW_REG_ZERO(rdpm, rdpm_pl); 10499 10500 /* HW is determining switch priority based on DSCP-bits, but the 10501 * kernel is still doing that based on the ToS. Since there's a 10502 * mismatch in bits we need to make sure to translate the right 10503 * value ToS would observe, skipping the 2 least-significant ECN bits. 10504 */ 10505 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 10506 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 10507 10508 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 10509 } 10510 10511 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 10512 { 10513 struct net *net = mlxsw_sp_net(mlxsw_sp); 10514 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 10515 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10516 u64 max_rifs; 10517 10518 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 10519 return -EIO; 10520 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10521 10522 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 10523 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 10524 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 10525 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10526 } 10527 10528 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10529 { 10530 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10531 10532 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 10533 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10534 } 10535 10536 static const struct mlxsw_sp_router_ll_ops mlxsw_sp_router_ll_basic_ops = { 10537 .init = mlxsw_sp_router_ll_basic_init, 10538 .ralta_write = mlxsw_sp_router_ll_basic_ralta_write, 10539 .ralst_write = mlxsw_sp_router_ll_basic_ralst_write, 10540 .raltb_write = mlxsw_sp_router_ll_basic_raltb_write, 10541 .fib_entry_op_ctx_size = sizeof(struct mlxsw_sp_fib_entry_op_ctx_basic), 10542 .fib_entry_pack = mlxsw_sp_router_ll_basic_fib_entry_pack, 10543 .fib_entry_act_remote_pack = mlxsw_sp_router_ll_basic_fib_entry_act_remote_pack, 10544 .fib_entry_act_local_pack = mlxsw_sp_router_ll_basic_fib_entry_act_local_pack, 10545 .fib_entry_act_ip2me_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_pack, 10546 .fib_entry_act_ip2me_tun_pack = mlxsw_sp_router_ll_basic_fib_entry_act_ip2me_tun_pack, 10547 .fib_entry_commit = mlxsw_sp_router_ll_basic_fib_entry_commit, 10548 .fib_entry_is_committed = mlxsw_sp_router_ll_basic_fib_entry_is_committed, 10549 }; 10550 10551 static int mlxsw_sp_router_ll_op_ctx_init(struct mlxsw_sp_router *router) 10552 { 10553 size_t max_size = 0; 10554 int i; 10555 10556 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 10557 size_t size = router->proto_ll_ops[i]->fib_entry_op_ctx_size; 10558 10559 if (size > max_size) 10560 max_size = size; 10561 } 10562 router->ll_op_ctx = kzalloc(sizeof(*router->ll_op_ctx) + max_size, 10563 GFP_KERNEL); 10564 if (!router->ll_op_ctx) 10565 return -ENOMEM; 10566 INIT_LIST_HEAD(&router->ll_op_ctx->fib_entry_priv_list); 10567 return 0; 10568 } 10569 10570 static void mlxsw_sp_router_ll_op_ctx_fini(struct mlxsw_sp_router *router) 10571 { 10572 WARN_ON(!list_empty(&router->ll_op_ctx->fib_entry_priv_list)); 10573 kfree(router->ll_op_ctx); 10574 } 10575 10576 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 10577 { 10578 u16 lb_rif_index; 10579 int err; 10580 10581 /* Create a generic loopback RIF associated with the main table 10582 * (default VRF). Any table can be used, but the main table exists 10583 * anyway, so we do not waste resources. 10584 */ 10585 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 10586 &lb_rif_index); 10587 if (err) 10588 return err; 10589 10590 mlxsw_sp->router->lb_rif_index = lb_rif_index; 10591 10592 return 0; 10593 } 10594 10595 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 10596 { 10597 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 10598 } 10599 10600 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 10601 { 10602 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 10603 10604 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 10605 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 10606 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10607 10608 return 0; 10609 } 10610 10611 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 10612 .init = mlxsw_sp1_router_init, 10613 .ipips_init = mlxsw_sp1_ipips_init, 10614 }; 10615 10616 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 10617 { 10618 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 10619 10620 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 10621 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 10622 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10623 10624 return 0; 10625 } 10626 10627 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 10628 .init = mlxsw_sp2_router_init, 10629 .ipips_init = mlxsw_sp2_ipips_init, 10630 }; 10631 10632 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 10633 struct netlink_ext_ack *extack) 10634 { 10635 struct mlxsw_sp_router *router; 10636 int err; 10637 10638 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 10639 if (!router) 10640 return -ENOMEM; 10641 mutex_init(&router->lock); 10642 mlxsw_sp->router = router; 10643 router->mlxsw_sp = mlxsw_sp; 10644 10645 err = mlxsw_sp->router_ops->init(mlxsw_sp); 10646 if (err) 10647 goto err_router_ops_init; 10648 10649 err = mlxsw_sp_router_xm_init(mlxsw_sp); 10650 if (err) 10651 goto err_xm_init; 10652 10653 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV4] = mlxsw_sp_router_xm_ipv4_is_supported(mlxsw_sp) ? 10654 &mlxsw_sp_router_ll_xm_ops : 10655 &mlxsw_sp_router_ll_basic_ops; 10656 router->proto_ll_ops[MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_router_ll_basic_ops; 10657 10658 err = mlxsw_sp_router_ll_op_ctx_init(router); 10659 if (err) 10660 goto err_ll_op_ctx_init; 10661 10662 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 10663 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 10664 mlxsw_sp_nh_grp_activity_work); 10665 10666 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 10667 err = __mlxsw_sp_router_init(mlxsw_sp); 10668 if (err) 10669 goto err_router_init; 10670 10671 err = mlxsw_sp_rifs_init(mlxsw_sp); 10672 if (err) 10673 goto err_rifs_init; 10674 10675 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 10676 if (err) 10677 goto err_ipips_init; 10678 10679 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 10680 &mlxsw_sp_nexthop_ht_params); 10681 if (err) 10682 goto err_nexthop_ht_init; 10683 10684 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 10685 &mlxsw_sp_nexthop_group_ht_params); 10686 if (err) 10687 goto err_nexthop_group_ht_init; 10688 10689 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 10690 err = mlxsw_sp_lpm_init(mlxsw_sp); 10691 if (err) 10692 goto err_lpm_init; 10693 10694 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 10695 if (err) 10696 goto err_mr_init; 10697 10698 err = mlxsw_sp_vrs_init(mlxsw_sp); 10699 if (err) 10700 goto err_vrs_init; 10701 10702 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 10703 if (err) 10704 goto err_lb_rif_init; 10705 10706 err = mlxsw_sp_neigh_init(mlxsw_sp); 10707 if (err) 10708 goto err_neigh_init; 10709 10710 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 10711 if (err) 10712 goto err_mp_hash_init; 10713 10714 err = mlxsw_sp_dscp_init(mlxsw_sp); 10715 if (err) 10716 goto err_dscp_init; 10717 10718 INIT_WORK(&router->fib_event_work, mlxsw_sp_router_fib_event_work); 10719 INIT_LIST_HEAD(&router->fib_event_queue); 10720 spin_lock_init(&router->fib_event_queue_lock); 10721 10722 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 10723 err = register_inetaddr_notifier(&router->inetaddr_nb); 10724 if (err) 10725 goto err_register_inetaddr_notifier; 10726 10727 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 10728 err = register_inet6addr_notifier(&router->inet6addr_nb); 10729 if (err) 10730 goto err_register_inet6addr_notifier; 10731 10732 mlxsw_sp->router->netevent_nb.notifier_call = 10733 mlxsw_sp_router_netevent_event; 10734 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10735 if (err) 10736 goto err_register_netevent_notifier; 10737 10738 mlxsw_sp->router->nexthop_nb.notifier_call = 10739 mlxsw_sp_nexthop_obj_event; 10740 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10741 &mlxsw_sp->router->nexthop_nb, 10742 extack); 10743 if (err) 10744 goto err_register_nexthop_notifier; 10745 10746 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 10747 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10748 &mlxsw_sp->router->fib_nb, 10749 mlxsw_sp_router_fib_dump_flush, extack); 10750 if (err) 10751 goto err_register_fib_notifier; 10752 10753 mlxsw_sp->router->netdevice_nb.notifier_call = 10754 mlxsw_sp_router_netdevice_event; 10755 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10756 &mlxsw_sp->router->netdevice_nb); 10757 if (err) 10758 goto err_register_netdev_notifier; 10759 10760 return 0; 10761 10762 err_register_netdev_notifier: 10763 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10764 &mlxsw_sp->router->fib_nb); 10765 err_register_fib_notifier: 10766 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10767 &mlxsw_sp->router->nexthop_nb); 10768 err_register_nexthop_notifier: 10769 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10770 err_register_netevent_notifier: 10771 unregister_inet6addr_notifier(&router->inet6addr_nb); 10772 err_register_inet6addr_notifier: 10773 unregister_inetaddr_notifier(&router->inetaddr_nb); 10774 err_register_inetaddr_notifier: 10775 mlxsw_core_flush_owq(); 10776 WARN_ON(!list_empty(&router->fib_event_queue)); 10777 err_dscp_init: 10778 err_mp_hash_init: 10779 mlxsw_sp_neigh_fini(mlxsw_sp); 10780 err_neigh_init: 10781 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10782 err_lb_rif_init: 10783 mlxsw_sp_vrs_fini(mlxsw_sp); 10784 err_vrs_init: 10785 mlxsw_sp_mr_fini(mlxsw_sp); 10786 err_mr_init: 10787 mlxsw_sp_lpm_fini(mlxsw_sp); 10788 err_lpm_init: 10789 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10790 err_nexthop_group_ht_init: 10791 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10792 err_nexthop_ht_init: 10793 mlxsw_sp_ipips_fini(mlxsw_sp); 10794 err_ipips_init: 10795 mlxsw_sp_rifs_fini(mlxsw_sp); 10796 err_rifs_init: 10797 __mlxsw_sp_router_fini(mlxsw_sp); 10798 err_router_init: 10799 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10800 mlxsw_sp_router_ll_op_ctx_fini(router); 10801 err_ll_op_ctx_init: 10802 mlxsw_sp_router_xm_fini(mlxsw_sp); 10803 err_xm_init: 10804 err_router_ops_init: 10805 mutex_destroy(&mlxsw_sp->router->lock); 10806 kfree(mlxsw_sp->router); 10807 return err; 10808 } 10809 10810 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10811 { 10812 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10813 &mlxsw_sp->router->netdevice_nb); 10814 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10815 &mlxsw_sp->router->fib_nb); 10816 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10817 &mlxsw_sp->router->nexthop_nb); 10818 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10819 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 10820 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 10821 mlxsw_core_flush_owq(); 10822 WARN_ON(!list_empty(&mlxsw_sp->router->fib_event_queue)); 10823 mlxsw_sp_neigh_fini(mlxsw_sp); 10824 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10825 mlxsw_sp_vrs_fini(mlxsw_sp); 10826 mlxsw_sp_mr_fini(mlxsw_sp); 10827 mlxsw_sp_lpm_fini(mlxsw_sp); 10828 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10829 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10830 mlxsw_sp_ipips_fini(mlxsw_sp); 10831 mlxsw_sp_rifs_fini(mlxsw_sp); 10832 __mlxsw_sp_router_fini(mlxsw_sp); 10833 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10834 mlxsw_sp_router_ll_op_ctx_fini(mlxsw_sp->router); 10835 mlxsw_sp_router_xm_fini(mlxsw_sp); 10836 mutex_destroy(&mlxsw_sp->router->lock); 10837 kfree(mlxsw_sp->router); 10838 } 10839