1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <linux/genalloc.h> 22 #include <net/netevent.h> 23 #include <net/neighbour.h> 24 #include <net/arp.h> 25 #include <net/inet_dscp.h> 26 #include <net/ip_fib.h> 27 #include <net/ip6_fib.h> 28 #include <net/nexthop.h> 29 #include <net/fib_rules.h> 30 #include <net/ip_tunnels.h> 31 #include <net/l3mdev.h> 32 #include <net/addrconf.h> 33 #include <net/ndisc.h> 34 #include <net/ipv6.h> 35 #include <net/fib_notifier.h> 36 #include <net/switchdev.h> 37 38 #include "spectrum.h" 39 #include "core.h" 40 #include "reg.h" 41 #include "spectrum_cnt.h" 42 #include "spectrum_dpipe.h" 43 #include "spectrum_ipip.h" 44 #include "spectrum_mr.h" 45 #include "spectrum_mr_tcam.h" 46 #include "spectrum_router.h" 47 #include "spectrum_span.h" 48 49 struct mlxsw_sp_fib; 50 struct mlxsw_sp_vr; 51 struct mlxsw_sp_lpm_tree; 52 struct mlxsw_sp_rif_ops; 53 54 struct mlxsw_sp_crif_key { 55 struct net_device *dev; 56 }; 57 58 struct mlxsw_sp_crif { 59 struct mlxsw_sp_crif_key key; 60 struct rhash_head ht_node; 61 bool can_destroy; 62 struct list_head nexthop_list; 63 struct mlxsw_sp_rif *rif; 64 }; 65 66 static const struct rhashtable_params mlxsw_sp_crif_ht_params = { 67 .key_offset = offsetof(struct mlxsw_sp_crif, key), 68 .key_len = sizeof_field(struct mlxsw_sp_crif, key), 69 .head_offset = offsetof(struct mlxsw_sp_crif, ht_node), 70 }; 71 72 struct mlxsw_sp_rif { 73 struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */ 74 struct list_head neigh_list; 75 struct mlxsw_sp_fid *fid; 76 unsigned char addr[ETH_ALEN]; 77 int mtu; 78 u16 rif_index; 79 u8 mac_profile_id; 80 u8 rif_entries; 81 u16 vr_id; 82 const struct mlxsw_sp_rif_ops *ops; 83 struct mlxsw_sp *mlxsw_sp; 84 85 unsigned int counter_ingress; 86 bool counter_ingress_valid; 87 unsigned int counter_egress; 88 bool counter_egress_valid; 89 }; 90 91 static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 92 { 93 if (!rif->crif) 94 return NULL; 95 return rif->crif->key.dev; 96 } 97 98 struct mlxsw_sp_rif_params { 99 struct net_device *dev; 100 union { 101 u16 system_port; 102 u16 lag_id; 103 }; 104 u16 vid; 105 bool lag; 106 bool double_entry; 107 }; 108 109 struct mlxsw_sp_rif_subport { 110 struct mlxsw_sp_rif common; 111 refcount_t ref_count; 112 union { 113 u16 system_port; 114 u16 lag_id; 115 }; 116 u16 vid; 117 bool lag; 118 }; 119 120 struct mlxsw_sp_rif_ipip_lb { 121 struct mlxsw_sp_rif common; 122 struct mlxsw_sp_rif_ipip_lb_config lb_config; 123 u16 ul_vr_id; /* Spectrum-1. */ 124 u16 ul_rif_id; /* Spectrum-2+. */ 125 }; 126 127 struct mlxsw_sp_rif_params_ipip_lb { 128 struct mlxsw_sp_rif_params common; 129 struct mlxsw_sp_rif_ipip_lb_config lb_config; 130 }; 131 132 struct mlxsw_sp_rif_ops { 133 enum mlxsw_sp_rif_type type; 134 size_t rif_size; 135 136 void (*setup)(struct mlxsw_sp_rif *rif, 137 const struct mlxsw_sp_rif_params *params); 138 int (*configure)(struct mlxsw_sp_rif *rif, 139 struct netlink_ext_ack *extack); 140 void (*deconfigure)(struct mlxsw_sp_rif *rif); 141 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 142 struct netlink_ext_ack *extack); 143 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 144 }; 145 146 struct mlxsw_sp_rif_mac_profile { 147 unsigned char mac_prefix[ETH_ALEN]; 148 refcount_t ref_count; 149 u8 id; 150 }; 151 152 struct mlxsw_sp_router_ops { 153 int (*init)(struct mlxsw_sp *mlxsw_sp); 154 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 155 }; 156 157 static struct mlxsw_sp_rif * 158 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 159 const struct net_device *dev); 160 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 161 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 162 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 163 struct mlxsw_sp_lpm_tree *lpm_tree); 164 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 165 const struct mlxsw_sp_fib *fib, 166 u8 tree_id); 167 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 168 const struct mlxsw_sp_fib *fib); 169 170 static unsigned int * 171 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 172 enum mlxsw_sp_rif_counter_dir dir) 173 { 174 switch (dir) { 175 case MLXSW_SP_RIF_COUNTER_EGRESS: 176 return &rif->counter_egress; 177 case MLXSW_SP_RIF_COUNTER_INGRESS: 178 return &rif->counter_ingress; 179 } 180 return NULL; 181 } 182 183 static bool 184 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 185 enum mlxsw_sp_rif_counter_dir dir) 186 { 187 switch (dir) { 188 case MLXSW_SP_RIF_COUNTER_EGRESS: 189 return rif->counter_egress_valid; 190 case MLXSW_SP_RIF_COUNTER_INGRESS: 191 return rif->counter_ingress_valid; 192 } 193 return false; 194 } 195 196 static void 197 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 198 enum mlxsw_sp_rif_counter_dir dir, 199 bool valid) 200 { 201 switch (dir) { 202 case MLXSW_SP_RIF_COUNTER_EGRESS: 203 rif->counter_egress_valid = valid; 204 break; 205 case MLXSW_SP_RIF_COUNTER_INGRESS: 206 rif->counter_ingress_valid = valid; 207 break; 208 } 209 } 210 211 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 212 unsigned int counter_index, bool enable, 213 enum mlxsw_sp_rif_counter_dir dir) 214 { 215 char ritr_pl[MLXSW_REG_RITR_LEN]; 216 bool is_egress = false; 217 int err; 218 219 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 220 is_egress = true; 221 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 222 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 223 if (err) 224 return err; 225 226 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 227 is_egress); 228 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 229 } 230 231 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 232 struct mlxsw_sp_rif *rif, 233 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 234 { 235 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 236 unsigned int *p_counter_index; 237 bool valid; 238 int err; 239 240 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 241 if (!valid) 242 return -EINVAL; 243 244 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 245 if (!p_counter_index) 246 return -EINVAL; 247 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 248 MLXSW_REG_RICNT_OPCODE_NOP); 249 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 250 if (err) 251 return err; 252 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 253 return 0; 254 } 255 256 struct mlxsw_sp_rif_counter_set_basic { 257 u64 good_unicast_packets; 258 u64 good_multicast_packets; 259 u64 good_broadcast_packets; 260 u64 good_unicast_bytes; 261 u64 good_multicast_bytes; 262 u64 good_broadcast_bytes; 263 u64 error_packets; 264 u64 discard_packets; 265 u64 error_bytes; 266 u64 discard_bytes; 267 }; 268 269 static int 270 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 271 enum mlxsw_sp_rif_counter_dir dir, 272 struct mlxsw_sp_rif_counter_set_basic *set) 273 { 274 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 275 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 276 unsigned int *p_counter_index; 277 int err; 278 279 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 280 return -EINVAL; 281 282 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 283 if (!p_counter_index) 284 return -EINVAL; 285 286 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 287 MLXSW_REG_RICNT_OPCODE_CLEAR); 288 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 289 if (err) 290 return err; 291 292 if (!set) 293 return 0; 294 295 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 296 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 297 298 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 299 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 300 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 301 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 302 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 303 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 304 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 305 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 306 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 307 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 308 309 #undef MLXSW_SP_RIF_COUNTER_EXTRACT 310 311 return 0; 312 } 313 314 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 315 unsigned int counter_index) 316 { 317 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 318 319 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 320 MLXSW_REG_RICNT_OPCODE_CLEAR); 321 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 322 } 323 324 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 325 enum mlxsw_sp_rif_counter_dir dir) 326 { 327 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 328 unsigned int *p_counter_index; 329 int err; 330 331 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 332 return 0; 333 334 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 335 if (!p_counter_index) 336 return -EINVAL; 337 338 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 339 p_counter_index); 340 if (err) 341 return err; 342 343 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 344 if (err) 345 goto err_counter_clear; 346 347 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 348 *p_counter_index, true, dir); 349 if (err) 350 goto err_counter_edit; 351 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 352 return 0; 353 354 err_counter_edit: 355 err_counter_clear: 356 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 357 *p_counter_index); 358 return err; 359 } 360 361 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 362 enum mlxsw_sp_rif_counter_dir dir) 363 { 364 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 365 unsigned int *p_counter_index; 366 367 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 368 return; 369 370 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 371 if (WARN_ON(!p_counter_index)) 372 return; 373 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 374 *p_counter_index, false, dir); 375 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 376 *p_counter_index); 377 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 378 } 379 380 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 381 { 382 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 383 struct devlink *devlink; 384 385 devlink = priv_to_devlink(mlxsw_sp->core); 386 if (!devlink_dpipe_table_counter_enabled(devlink, 387 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 388 return; 389 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 390 } 391 392 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 393 { 394 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 395 } 396 397 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 398 399 struct mlxsw_sp_prefix_usage { 400 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 401 }; 402 403 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 404 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 405 406 static bool 407 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 408 struct mlxsw_sp_prefix_usage *prefix_usage2) 409 { 410 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 411 } 412 413 static void 414 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 415 struct mlxsw_sp_prefix_usage *prefix_usage2) 416 { 417 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 418 } 419 420 static void 421 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 422 unsigned char prefix_len) 423 { 424 set_bit(prefix_len, prefix_usage->b); 425 } 426 427 static void 428 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 429 unsigned char prefix_len) 430 { 431 clear_bit(prefix_len, prefix_usage->b); 432 } 433 434 struct mlxsw_sp_fib_key { 435 unsigned char addr[sizeof(struct in6_addr)]; 436 unsigned char prefix_len; 437 }; 438 439 enum mlxsw_sp_fib_entry_type { 440 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 441 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 442 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 443 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 444 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 445 446 /* This is a special case of local delivery, where a packet should be 447 * decapsulated on reception. Note that there is no corresponding ENCAP, 448 * because that's a type of next hop, not of FIB entry. (There can be 449 * several next hops in a REMOTE entry, and some of them may be 450 * encapsulating entries.) 451 */ 452 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 453 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 454 }; 455 456 struct mlxsw_sp_nexthop_group_info; 457 struct mlxsw_sp_nexthop_group; 458 struct mlxsw_sp_fib_entry; 459 460 struct mlxsw_sp_fib_node { 461 struct mlxsw_sp_fib_entry *fib_entry; 462 struct list_head list; 463 struct rhash_head ht_node; 464 struct mlxsw_sp_fib *fib; 465 struct mlxsw_sp_fib_key key; 466 }; 467 468 struct mlxsw_sp_fib_entry_decap { 469 struct mlxsw_sp_ipip_entry *ipip_entry; 470 u32 tunnel_index; 471 }; 472 473 struct mlxsw_sp_fib_entry { 474 struct mlxsw_sp_fib_node *fib_node; 475 enum mlxsw_sp_fib_entry_type type; 476 struct list_head nexthop_group_node; 477 struct mlxsw_sp_nexthop_group *nh_group; 478 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 479 }; 480 481 struct mlxsw_sp_fib4_entry { 482 struct mlxsw_sp_fib_entry common; 483 struct fib_info *fi; 484 u32 tb_id; 485 dscp_t dscp; 486 u8 type; 487 }; 488 489 struct mlxsw_sp_fib6_entry { 490 struct mlxsw_sp_fib_entry common; 491 struct list_head rt6_list; 492 unsigned int nrt6; 493 }; 494 495 struct mlxsw_sp_rt6 { 496 struct list_head list; 497 struct fib6_info *rt; 498 }; 499 500 struct mlxsw_sp_lpm_tree { 501 u8 id; /* tree ID */ 502 unsigned int ref_count; 503 enum mlxsw_sp_l3proto proto; 504 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 505 struct mlxsw_sp_prefix_usage prefix_usage; 506 }; 507 508 struct mlxsw_sp_fib { 509 struct rhashtable ht; 510 struct list_head node_list; 511 struct mlxsw_sp_vr *vr; 512 struct mlxsw_sp_lpm_tree *lpm_tree; 513 enum mlxsw_sp_l3proto proto; 514 }; 515 516 struct mlxsw_sp_vr { 517 u16 id; /* virtual router ID */ 518 u32 tb_id; /* kernel fib table id */ 519 unsigned int rif_count; 520 struct mlxsw_sp_fib *fib4; 521 struct mlxsw_sp_fib *fib6; 522 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 523 struct mlxsw_sp_rif *ul_rif; 524 refcount_t ul_rif_refcnt; 525 }; 526 527 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 528 529 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 530 struct mlxsw_sp_vr *vr, 531 enum mlxsw_sp_l3proto proto) 532 { 533 struct mlxsw_sp_lpm_tree *lpm_tree; 534 struct mlxsw_sp_fib *fib; 535 int err; 536 537 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 538 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 539 if (!fib) 540 return ERR_PTR(-ENOMEM); 541 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 542 if (err) 543 goto err_rhashtable_init; 544 INIT_LIST_HEAD(&fib->node_list); 545 fib->proto = proto; 546 fib->vr = vr; 547 fib->lpm_tree = lpm_tree; 548 mlxsw_sp_lpm_tree_hold(lpm_tree); 549 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 550 if (err) 551 goto err_lpm_tree_bind; 552 return fib; 553 554 err_lpm_tree_bind: 555 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 556 err_rhashtable_init: 557 kfree(fib); 558 return ERR_PTR(err); 559 } 560 561 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 562 struct mlxsw_sp_fib *fib) 563 { 564 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 565 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 566 WARN_ON(!list_empty(&fib->node_list)); 567 rhashtable_destroy(&fib->ht); 568 kfree(fib); 569 } 570 571 static struct mlxsw_sp_lpm_tree * 572 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 573 { 574 static struct mlxsw_sp_lpm_tree *lpm_tree; 575 int i; 576 577 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 578 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 579 if (lpm_tree->ref_count == 0) 580 return lpm_tree; 581 } 582 return NULL; 583 } 584 585 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 586 struct mlxsw_sp_lpm_tree *lpm_tree) 587 { 588 char ralta_pl[MLXSW_REG_RALTA_LEN]; 589 590 mlxsw_reg_ralta_pack(ralta_pl, true, 591 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 592 lpm_tree->id); 593 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 594 } 595 596 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 597 struct mlxsw_sp_lpm_tree *lpm_tree) 598 { 599 char ralta_pl[MLXSW_REG_RALTA_LEN]; 600 601 mlxsw_reg_ralta_pack(ralta_pl, false, 602 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 603 lpm_tree->id); 604 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 605 } 606 607 static int 608 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 609 struct mlxsw_sp_prefix_usage *prefix_usage, 610 struct mlxsw_sp_lpm_tree *lpm_tree) 611 { 612 char ralst_pl[MLXSW_REG_RALST_LEN]; 613 u8 root_bin = 0; 614 u8 prefix; 615 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 616 617 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 618 root_bin = prefix; 619 620 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 621 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 622 if (prefix == 0) 623 continue; 624 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 625 MLXSW_REG_RALST_BIN_NO_CHILD); 626 last_prefix = prefix; 627 } 628 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 629 } 630 631 static struct mlxsw_sp_lpm_tree * 632 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 633 struct mlxsw_sp_prefix_usage *prefix_usage, 634 enum mlxsw_sp_l3proto proto) 635 { 636 struct mlxsw_sp_lpm_tree *lpm_tree; 637 int err; 638 639 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 640 if (!lpm_tree) 641 return ERR_PTR(-EBUSY); 642 lpm_tree->proto = proto; 643 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 644 if (err) 645 return ERR_PTR(err); 646 647 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 648 lpm_tree); 649 if (err) 650 goto err_left_struct_set; 651 memcpy(&lpm_tree->prefix_usage, prefix_usage, 652 sizeof(lpm_tree->prefix_usage)); 653 memset(&lpm_tree->prefix_ref_count, 0, 654 sizeof(lpm_tree->prefix_ref_count)); 655 lpm_tree->ref_count = 1; 656 return lpm_tree; 657 658 err_left_struct_set: 659 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 660 return ERR_PTR(err); 661 } 662 663 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 664 struct mlxsw_sp_lpm_tree *lpm_tree) 665 { 666 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 667 } 668 669 static struct mlxsw_sp_lpm_tree * 670 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 671 struct mlxsw_sp_prefix_usage *prefix_usage, 672 enum mlxsw_sp_l3proto proto) 673 { 674 struct mlxsw_sp_lpm_tree *lpm_tree; 675 int i; 676 677 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 678 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 679 if (lpm_tree->ref_count != 0 && 680 lpm_tree->proto == proto && 681 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 682 prefix_usage)) { 683 mlxsw_sp_lpm_tree_hold(lpm_tree); 684 return lpm_tree; 685 } 686 } 687 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 688 } 689 690 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 691 { 692 lpm_tree->ref_count++; 693 } 694 695 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 696 struct mlxsw_sp_lpm_tree *lpm_tree) 697 { 698 if (--lpm_tree->ref_count == 0) 699 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 700 } 701 702 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 703 704 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 705 { 706 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 707 struct mlxsw_sp_lpm_tree *lpm_tree; 708 u64 max_trees; 709 int err, i; 710 711 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 712 return -EIO; 713 714 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 715 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 716 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 717 sizeof(struct mlxsw_sp_lpm_tree), 718 GFP_KERNEL); 719 if (!mlxsw_sp->router->lpm.trees) 720 return -ENOMEM; 721 722 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 723 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 724 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 725 } 726 727 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 728 MLXSW_SP_L3_PROTO_IPV4); 729 if (IS_ERR(lpm_tree)) { 730 err = PTR_ERR(lpm_tree); 731 goto err_ipv4_tree_get; 732 } 733 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 734 735 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 736 MLXSW_SP_L3_PROTO_IPV6); 737 if (IS_ERR(lpm_tree)) { 738 err = PTR_ERR(lpm_tree); 739 goto err_ipv6_tree_get; 740 } 741 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 742 743 return 0; 744 745 err_ipv6_tree_get: 746 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 747 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 748 err_ipv4_tree_get: 749 kfree(mlxsw_sp->router->lpm.trees); 750 return err; 751 } 752 753 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 754 { 755 struct mlxsw_sp_lpm_tree *lpm_tree; 756 757 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 758 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 759 760 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 761 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 762 763 kfree(mlxsw_sp->router->lpm.trees); 764 } 765 766 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 767 { 768 return !!vr->fib4 || !!vr->fib6 || 769 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 770 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 771 } 772 773 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 774 { 775 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 776 struct mlxsw_sp_vr *vr; 777 int i; 778 779 for (i = 0; i < max_vrs; i++) { 780 vr = &mlxsw_sp->router->vrs[i]; 781 if (!mlxsw_sp_vr_is_used(vr)) 782 return vr; 783 } 784 return NULL; 785 } 786 787 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 788 const struct mlxsw_sp_fib *fib, u8 tree_id) 789 { 790 char raltb_pl[MLXSW_REG_RALTB_LEN]; 791 792 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 793 (enum mlxsw_reg_ralxx_protocol) fib->proto, 794 tree_id); 795 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 796 } 797 798 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 799 const struct mlxsw_sp_fib *fib) 800 { 801 char raltb_pl[MLXSW_REG_RALTB_LEN]; 802 803 /* Bind to tree 0 which is default */ 804 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 805 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 806 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 807 } 808 809 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 810 { 811 /* For our purpose, squash main, default and local tables into one */ 812 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 813 tb_id = RT_TABLE_MAIN; 814 return tb_id; 815 } 816 817 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 818 u32 tb_id) 819 { 820 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 821 struct mlxsw_sp_vr *vr; 822 int i; 823 824 tb_id = mlxsw_sp_fix_tb_id(tb_id); 825 826 for (i = 0; i < max_vrs; i++) { 827 vr = &mlxsw_sp->router->vrs[i]; 828 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 829 return vr; 830 } 831 return NULL; 832 } 833 834 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 835 u16 *vr_id) 836 { 837 struct mlxsw_sp_vr *vr; 838 int err = 0; 839 840 mutex_lock(&mlxsw_sp->router->lock); 841 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 842 if (!vr) { 843 err = -ESRCH; 844 goto out; 845 } 846 *vr_id = vr->id; 847 out: 848 mutex_unlock(&mlxsw_sp->router->lock); 849 return err; 850 } 851 852 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 853 enum mlxsw_sp_l3proto proto) 854 { 855 switch (proto) { 856 case MLXSW_SP_L3_PROTO_IPV4: 857 return vr->fib4; 858 case MLXSW_SP_L3_PROTO_IPV6: 859 return vr->fib6; 860 } 861 return NULL; 862 } 863 864 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 865 u32 tb_id, 866 struct netlink_ext_ack *extack) 867 { 868 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 869 struct mlxsw_sp_fib *fib4; 870 struct mlxsw_sp_fib *fib6; 871 struct mlxsw_sp_vr *vr; 872 int err; 873 874 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 875 if (!vr) { 876 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 877 return ERR_PTR(-EBUSY); 878 } 879 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 880 if (IS_ERR(fib4)) 881 return ERR_CAST(fib4); 882 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 883 if (IS_ERR(fib6)) { 884 err = PTR_ERR(fib6); 885 goto err_fib6_create; 886 } 887 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 888 MLXSW_SP_L3_PROTO_IPV4); 889 if (IS_ERR(mr4_table)) { 890 err = PTR_ERR(mr4_table); 891 goto err_mr4_table_create; 892 } 893 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 894 MLXSW_SP_L3_PROTO_IPV6); 895 if (IS_ERR(mr6_table)) { 896 err = PTR_ERR(mr6_table); 897 goto err_mr6_table_create; 898 } 899 900 vr->fib4 = fib4; 901 vr->fib6 = fib6; 902 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 903 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 904 vr->tb_id = tb_id; 905 return vr; 906 907 err_mr6_table_create: 908 mlxsw_sp_mr_table_destroy(mr4_table); 909 err_mr4_table_create: 910 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 911 err_fib6_create: 912 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 913 return ERR_PTR(err); 914 } 915 916 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 917 struct mlxsw_sp_vr *vr) 918 { 919 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 920 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 921 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 922 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 923 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 924 vr->fib6 = NULL; 925 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 926 vr->fib4 = NULL; 927 } 928 929 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 930 struct netlink_ext_ack *extack) 931 { 932 struct mlxsw_sp_vr *vr; 933 934 tb_id = mlxsw_sp_fix_tb_id(tb_id); 935 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 936 if (!vr) 937 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 938 return vr; 939 } 940 941 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 942 { 943 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 944 list_empty(&vr->fib6->node_list) && 945 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 946 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 947 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 948 } 949 950 static bool 951 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 952 enum mlxsw_sp_l3proto proto, u8 tree_id) 953 { 954 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 955 956 if (!mlxsw_sp_vr_is_used(vr)) 957 return false; 958 if (fib->lpm_tree->id == tree_id) 959 return true; 960 return false; 961 } 962 963 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 964 struct mlxsw_sp_fib *fib, 965 struct mlxsw_sp_lpm_tree *new_tree) 966 { 967 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 968 int err; 969 970 fib->lpm_tree = new_tree; 971 mlxsw_sp_lpm_tree_hold(new_tree); 972 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 973 if (err) 974 goto err_tree_bind; 975 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 976 return 0; 977 978 err_tree_bind: 979 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 980 fib->lpm_tree = old_tree; 981 return err; 982 } 983 984 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 985 struct mlxsw_sp_fib *fib, 986 struct mlxsw_sp_lpm_tree *new_tree) 987 { 988 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 989 enum mlxsw_sp_l3proto proto = fib->proto; 990 struct mlxsw_sp_lpm_tree *old_tree; 991 u8 old_id, new_id = new_tree->id; 992 struct mlxsw_sp_vr *vr; 993 int i, err; 994 995 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 996 old_id = old_tree->id; 997 998 for (i = 0; i < max_vrs; i++) { 999 vr = &mlxsw_sp->router->vrs[i]; 1000 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 1001 continue; 1002 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1003 mlxsw_sp_vr_fib(vr, proto), 1004 new_tree); 1005 if (err) 1006 goto err_tree_replace; 1007 } 1008 1009 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 1010 sizeof(new_tree->prefix_ref_count)); 1011 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 1012 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 1013 1014 return 0; 1015 1016 err_tree_replace: 1017 for (i--; i >= 0; i--) { 1018 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 1019 continue; 1020 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 1021 mlxsw_sp_vr_fib(vr, proto), 1022 old_tree); 1023 } 1024 return err; 1025 } 1026 1027 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 1028 { 1029 struct mlxsw_sp_vr *vr; 1030 u64 max_vrs; 1031 int i; 1032 1033 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1034 return -EIO; 1035 1036 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1037 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1038 GFP_KERNEL); 1039 if (!mlxsw_sp->router->vrs) 1040 return -ENOMEM; 1041 1042 for (i = 0; i < max_vrs; i++) { 1043 vr = &mlxsw_sp->router->vrs[i]; 1044 vr->id = i; 1045 } 1046 1047 return 0; 1048 } 1049 1050 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1051 1052 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1053 { 1054 /* At this stage we're guaranteed not to have new incoming 1055 * FIB notifications and the work queue is free from FIBs 1056 * sitting on top of mlxsw netdevs. However, we can still 1057 * have other FIBs queued. Flush the queue before flushing 1058 * the device's tables. No need for locks, as we're the only 1059 * writer. 1060 */ 1061 mlxsw_core_flush_owq(); 1062 mlxsw_sp_router_fib_flush(mlxsw_sp); 1063 kfree(mlxsw_sp->router->vrs); 1064 } 1065 1066 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1067 { 1068 struct net_device *d; 1069 u32 tb_id; 1070 1071 rcu_read_lock(); 1072 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1073 if (d) 1074 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1075 else 1076 tb_id = RT_TABLE_MAIN; 1077 rcu_read_unlock(); 1078 1079 return tb_id; 1080 } 1081 1082 static void 1083 mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev) 1084 { 1085 crif->key.dev = dev; 1086 INIT_LIST_HEAD(&crif->nexthop_list); 1087 } 1088 1089 static struct mlxsw_sp_crif * 1090 mlxsw_sp_crif_alloc(struct net_device *dev) 1091 { 1092 struct mlxsw_sp_crif *crif; 1093 1094 crif = kzalloc(sizeof(*crif), GFP_KERNEL); 1095 if (!crif) 1096 return NULL; 1097 1098 mlxsw_sp_crif_init(crif, dev); 1099 return crif; 1100 } 1101 1102 static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif) 1103 { 1104 if (WARN_ON(crif->rif)) 1105 return; 1106 1107 WARN_ON(!list_empty(&crif->nexthop_list)); 1108 kfree(crif); 1109 } 1110 1111 static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router, 1112 struct mlxsw_sp_crif *crif) 1113 { 1114 return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node, 1115 mlxsw_sp_crif_ht_params); 1116 } 1117 1118 static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router, 1119 struct mlxsw_sp_crif *crif) 1120 { 1121 rhashtable_remove_fast(&router->crif_ht, &crif->ht_node, 1122 mlxsw_sp_crif_ht_params); 1123 } 1124 1125 static struct mlxsw_sp_crif * 1126 mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router, 1127 const struct net_device *dev) 1128 { 1129 struct mlxsw_sp_crif_key key = { 1130 .dev = (struct net_device *)dev, 1131 }; 1132 1133 return rhashtable_lookup_fast(&router->crif_ht, &key, 1134 mlxsw_sp_crif_ht_params); 1135 } 1136 1137 static struct mlxsw_sp_rif * 1138 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1139 const struct mlxsw_sp_rif_params *params, 1140 struct netlink_ext_ack *extack); 1141 1142 static struct mlxsw_sp_rif_ipip_lb * 1143 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1144 enum mlxsw_sp_ipip_type ipipt, 1145 struct net_device *ol_dev, 1146 struct netlink_ext_ack *extack) 1147 { 1148 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1149 const struct mlxsw_sp_ipip_ops *ipip_ops; 1150 struct mlxsw_sp_rif *rif; 1151 1152 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1153 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1154 .common.dev = ol_dev, 1155 .common.lag = false, 1156 .common.double_entry = ipip_ops->double_rif_entry, 1157 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1158 }; 1159 1160 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1161 if (IS_ERR(rif)) 1162 return ERR_CAST(rif); 1163 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1164 } 1165 1166 static struct mlxsw_sp_ipip_entry * 1167 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1168 enum mlxsw_sp_ipip_type ipipt, 1169 struct net_device *ol_dev) 1170 { 1171 const struct mlxsw_sp_ipip_ops *ipip_ops; 1172 struct mlxsw_sp_ipip_entry *ipip_entry; 1173 struct mlxsw_sp_ipip_entry *ret = NULL; 1174 int err; 1175 1176 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1177 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1178 if (!ipip_entry) 1179 return ERR_PTR(-ENOMEM); 1180 1181 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1182 ol_dev, NULL); 1183 if (IS_ERR(ipip_entry->ol_lb)) { 1184 ret = ERR_CAST(ipip_entry->ol_lb); 1185 goto err_ol_ipip_lb_create; 1186 } 1187 1188 ipip_entry->ipipt = ipipt; 1189 ipip_entry->ol_dev = ol_dev; 1190 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1191 1192 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1193 if (err) { 1194 ret = ERR_PTR(err); 1195 goto err_rem_ip_addr_set; 1196 } 1197 1198 return ipip_entry; 1199 1200 err_rem_ip_addr_set: 1201 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1202 err_ol_ipip_lb_create: 1203 kfree(ipip_entry); 1204 return ret; 1205 } 1206 1207 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1208 struct mlxsw_sp_ipip_entry *ipip_entry) 1209 { 1210 const struct mlxsw_sp_ipip_ops *ipip_ops = 1211 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1212 1213 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1214 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1215 kfree(ipip_entry); 1216 } 1217 1218 static bool 1219 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1220 const enum mlxsw_sp_l3proto ul_proto, 1221 union mlxsw_sp_l3addr saddr, 1222 u32 ul_tb_id, 1223 struct mlxsw_sp_ipip_entry *ipip_entry) 1224 { 1225 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1226 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1227 union mlxsw_sp_l3addr tun_saddr; 1228 1229 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1230 return false; 1231 1232 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1233 return tun_ul_tb_id == ul_tb_id && 1234 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1235 } 1236 1237 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1238 enum mlxsw_sp_ipip_type ipipt) 1239 { 1240 const struct mlxsw_sp_ipip_ops *ipip_ops; 1241 1242 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1243 1244 /* Not all tunnels require to increase the default pasing depth 1245 * (96 bytes). 1246 */ 1247 if (ipip_ops->inc_parsing_depth) 1248 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1249 1250 return 0; 1251 } 1252 1253 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1254 enum mlxsw_sp_ipip_type ipipt) 1255 { 1256 const struct mlxsw_sp_ipip_ops *ipip_ops = 1257 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1258 1259 if (ipip_ops->inc_parsing_depth) 1260 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1261 } 1262 1263 static int 1264 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1265 struct mlxsw_sp_fib_entry *fib_entry, 1266 struct mlxsw_sp_ipip_entry *ipip_entry) 1267 { 1268 u32 tunnel_index; 1269 int err; 1270 1271 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1272 1, &tunnel_index); 1273 if (err) 1274 return err; 1275 1276 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1277 ipip_entry->ipipt); 1278 if (err) 1279 goto err_parsing_depth_inc; 1280 1281 ipip_entry->decap_fib_entry = fib_entry; 1282 fib_entry->decap.ipip_entry = ipip_entry; 1283 fib_entry->decap.tunnel_index = tunnel_index; 1284 1285 return 0; 1286 1287 err_parsing_depth_inc: 1288 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1289 fib_entry->decap.tunnel_index); 1290 return err; 1291 } 1292 1293 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1294 struct mlxsw_sp_fib_entry *fib_entry) 1295 { 1296 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1297 1298 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1299 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1300 fib_entry->decap.ipip_entry = NULL; 1301 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1302 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1303 1, fib_entry->decap.tunnel_index); 1304 } 1305 1306 static struct mlxsw_sp_fib_node * 1307 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1308 size_t addr_len, unsigned char prefix_len); 1309 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1310 struct mlxsw_sp_fib_entry *fib_entry); 1311 1312 static void 1313 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1314 struct mlxsw_sp_ipip_entry *ipip_entry) 1315 { 1316 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1317 1318 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1319 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1320 1321 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1322 } 1323 1324 static void 1325 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1326 struct mlxsw_sp_ipip_entry *ipip_entry, 1327 struct mlxsw_sp_fib_entry *decap_fib_entry) 1328 { 1329 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1330 ipip_entry)) 1331 return; 1332 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1333 1334 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1335 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1336 } 1337 1338 static struct mlxsw_sp_fib_entry * 1339 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1340 enum mlxsw_sp_l3proto proto, 1341 const union mlxsw_sp_l3addr *addr, 1342 enum mlxsw_sp_fib_entry_type type) 1343 { 1344 struct mlxsw_sp_fib_node *fib_node; 1345 unsigned char addr_prefix_len; 1346 struct mlxsw_sp_fib *fib; 1347 struct mlxsw_sp_vr *vr; 1348 const void *addrp; 1349 size_t addr_len; 1350 u32 addr4; 1351 1352 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1353 if (!vr) 1354 return NULL; 1355 fib = mlxsw_sp_vr_fib(vr, proto); 1356 1357 switch (proto) { 1358 case MLXSW_SP_L3_PROTO_IPV4: 1359 addr4 = be32_to_cpu(addr->addr4); 1360 addrp = &addr4; 1361 addr_len = 4; 1362 addr_prefix_len = 32; 1363 break; 1364 case MLXSW_SP_L3_PROTO_IPV6: 1365 addrp = &addr->addr6; 1366 addr_len = 16; 1367 addr_prefix_len = 128; 1368 break; 1369 default: 1370 WARN_ON(1); 1371 return NULL; 1372 } 1373 1374 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1375 addr_prefix_len); 1376 if (!fib_node || fib_node->fib_entry->type != type) 1377 return NULL; 1378 1379 return fib_node->fib_entry; 1380 } 1381 1382 /* Given an IPIP entry, find the corresponding decap route. */ 1383 static struct mlxsw_sp_fib_entry * 1384 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1385 struct mlxsw_sp_ipip_entry *ipip_entry) 1386 { 1387 static struct mlxsw_sp_fib_node *fib_node; 1388 const struct mlxsw_sp_ipip_ops *ipip_ops; 1389 unsigned char saddr_prefix_len; 1390 union mlxsw_sp_l3addr saddr; 1391 struct mlxsw_sp_fib *ul_fib; 1392 struct mlxsw_sp_vr *ul_vr; 1393 const void *saddrp; 1394 size_t saddr_len; 1395 u32 ul_tb_id; 1396 u32 saddr4; 1397 1398 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1399 1400 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1401 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1402 if (!ul_vr) 1403 return NULL; 1404 1405 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1406 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1407 ipip_entry->ol_dev); 1408 1409 switch (ipip_ops->ul_proto) { 1410 case MLXSW_SP_L3_PROTO_IPV4: 1411 saddr4 = be32_to_cpu(saddr.addr4); 1412 saddrp = &saddr4; 1413 saddr_len = 4; 1414 saddr_prefix_len = 32; 1415 break; 1416 case MLXSW_SP_L3_PROTO_IPV6: 1417 saddrp = &saddr.addr6; 1418 saddr_len = 16; 1419 saddr_prefix_len = 128; 1420 break; 1421 default: 1422 WARN_ON(1); 1423 return NULL; 1424 } 1425 1426 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1427 saddr_prefix_len); 1428 if (!fib_node || 1429 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1430 return NULL; 1431 1432 return fib_node->fib_entry; 1433 } 1434 1435 static struct mlxsw_sp_ipip_entry * 1436 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1437 enum mlxsw_sp_ipip_type ipipt, 1438 struct net_device *ol_dev) 1439 { 1440 struct mlxsw_sp_ipip_entry *ipip_entry; 1441 1442 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1443 if (IS_ERR(ipip_entry)) 1444 return ipip_entry; 1445 1446 list_add_tail(&ipip_entry->ipip_list_node, 1447 &mlxsw_sp->router->ipip_list); 1448 1449 return ipip_entry; 1450 } 1451 1452 static void 1453 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1454 struct mlxsw_sp_ipip_entry *ipip_entry) 1455 { 1456 list_del(&ipip_entry->ipip_list_node); 1457 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1458 } 1459 1460 static bool 1461 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1462 const struct net_device *ul_dev, 1463 enum mlxsw_sp_l3proto ul_proto, 1464 union mlxsw_sp_l3addr ul_dip, 1465 struct mlxsw_sp_ipip_entry *ipip_entry) 1466 { 1467 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1468 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1469 1470 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1471 return false; 1472 1473 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1474 ul_tb_id, ipip_entry); 1475 } 1476 1477 /* Given decap parameters, find the corresponding IPIP entry. */ 1478 static struct mlxsw_sp_ipip_entry * 1479 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1480 enum mlxsw_sp_l3proto ul_proto, 1481 union mlxsw_sp_l3addr ul_dip) 1482 { 1483 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1484 struct net_device *ul_dev; 1485 1486 rcu_read_lock(); 1487 1488 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1489 if (!ul_dev) 1490 goto out_unlock; 1491 1492 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1493 ipip_list_node) 1494 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1495 ul_proto, ul_dip, 1496 ipip_entry)) 1497 goto out_unlock; 1498 1499 rcu_read_unlock(); 1500 1501 return NULL; 1502 1503 out_unlock: 1504 rcu_read_unlock(); 1505 return ipip_entry; 1506 } 1507 1508 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1509 const struct net_device *dev, 1510 enum mlxsw_sp_ipip_type *p_type) 1511 { 1512 struct mlxsw_sp_router *router = mlxsw_sp->router; 1513 const struct mlxsw_sp_ipip_ops *ipip_ops; 1514 enum mlxsw_sp_ipip_type ipipt; 1515 1516 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1517 ipip_ops = router->ipip_ops_arr[ipipt]; 1518 if (dev->type == ipip_ops->dev_type) { 1519 if (p_type) 1520 *p_type = ipipt; 1521 return true; 1522 } 1523 } 1524 return false; 1525 } 1526 1527 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1528 const struct net_device *dev) 1529 { 1530 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1531 } 1532 1533 static struct mlxsw_sp_ipip_entry * 1534 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1535 const struct net_device *ol_dev) 1536 { 1537 struct mlxsw_sp_ipip_entry *ipip_entry; 1538 1539 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1540 ipip_list_node) 1541 if (ipip_entry->ol_dev == ol_dev) 1542 return ipip_entry; 1543 1544 return NULL; 1545 } 1546 1547 static struct mlxsw_sp_ipip_entry * 1548 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1549 const struct net_device *ul_dev, 1550 struct mlxsw_sp_ipip_entry *start) 1551 { 1552 struct mlxsw_sp_ipip_entry *ipip_entry; 1553 1554 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1555 ipip_list_node); 1556 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1557 ipip_list_node) { 1558 struct net_device *ol_dev = ipip_entry->ol_dev; 1559 struct net_device *ipip_ul_dev; 1560 1561 rcu_read_lock(); 1562 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1563 rcu_read_unlock(); 1564 1565 if (ipip_ul_dev == ul_dev) 1566 return ipip_entry; 1567 } 1568 1569 return NULL; 1570 } 1571 1572 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1573 const struct net_device *dev) 1574 { 1575 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1576 } 1577 1578 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1579 const struct net_device *ol_dev, 1580 enum mlxsw_sp_ipip_type ipipt) 1581 { 1582 const struct mlxsw_sp_ipip_ops *ops 1583 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1584 1585 return ops->can_offload(mlxsw_sp, ol_dev); 1586 } 1587 1588 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1589 struct net_device *ol_dev) 1590 { 1591 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1592 struct mlxsw_sp_ipip_entry *ipip_entry; 1593 enum mlxsw_sp_l3proto ul_proto; 1594 union mlxsw_sp_l3addr saddr; 1595 u32 ul_tb_id; 1596 1597 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1598 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1599 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1600 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1601 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1602 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1603 saddr, ul_tb_id, 1604 NULL)) { 1605 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1606 ol_dev); 1607 if (IS_ERR(ipip_entry)) 1608 return PTR_ERR(ipip_entry); 1609 } 1610 } 1611 1612 return 0; 1613 } 1614 1615 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1616 struct net_device *ol_dev) 1617 { 1618 struct mlxsw_sp_ipip_entry *ipip_entry; 1619 1620 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1621 if (ipip_entry) 1622 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1623 } 1624 1625 static void 1626 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1627 struct mlxsw_sp_ipip_entry *ipip_entry) 1628 { 1629 struct mlxsw_sp_fib_entry *decap_fib_entry; 1630 1631 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1632 if (decap_fib_entry) 1633 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1634 decap_fib_entry); 1635 } 1636 1637 static int 1638 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1639 u16 ul_rif_id, bool enable) 1640 { 1641 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1642 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 1643 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1644 struct mlxsw_sp_rif *rif = &lb_rif->common; 1645 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1646 char ritr_pl[MLXSW_REG_RITR_LEN]; 1647 struct in6_addr *saddr6; 1648 u32 saddr4; 1649 1650 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1651 switch (lb_cf.ul_protocol) { 1652 case MLXSW_SP_L3_PROTO_IPV4: 1653 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1654 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1655 rif->rif_index, rif->vr_id, dev->mtu); 1656 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1657 ipip_options, ul_vr_id, 1658 ul_rif_id, saddr4, 1659 lb_cf.okey); 1660 break; 1661 1662 case MLXSW_SP_L3_PROTO_IPV6: 1663 saddr6 = &lb_cf.saddr.addr6; 1664 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1665 rif->rif_index, rif->vr_id, dev->mtu); 1666 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1667 ipip_options, ul_vr_id, 1668 ul_rif_id, saddr6, 1669 lb_cf.okey); 1670 break; 1671 } 1672 1673 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1674 } 1675 1676 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1677 struct net_device *ol_dev) 1678 { 1679 struct mlxsw_sp_ipip_entry *ipip_entry; 1680 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1681 int err = 0; 1682 1683 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1684 if (ipip_entry) { 1685 lb_rif = ipip_entry->ol_lb; 1686 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1687 lb_rif->ul_rif_id, true); 1688 if (err) 1689 goto out; 1690 lb_rif->common.mtu = ol_dev->mtu; 1691 } 1692 1693 out: 1694 return err; 1695 } 1696 1697 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1698 struct net_device *ol_dev) 1699 { 1700 struct mlxsw_sp_ipip_entry *ipip_entry; 1701 1702 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1703 if (ipip_entry) 1704 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1705 } 1706 1707 static void 1708 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1709 struct mlxsw_sp_ipip_entry *ipip_entry) 1710 { 1711 if (ipip_entry->decap_fib_entry) 1712 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1713 } 1714 1715 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1716 struct net_device *ol_dev) 1717 { 1718 struct mlxsw_sp_ipip_entry *ipip_entry; 1719 1720 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1721 if (ipip_entry) 1722 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1723 } 1724 1725 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1726 struct mlxsw_sp_rif *rif); 1727 1728 static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp, 1729 struct mlxsw_sp_rif *old_rif, 1730 struct mlxsw_sp_rif *new_rif, 1731 bool migrate_nhs) 1732 { 1733 struct mlxsw_sp_crif *crif = old_rif->crif; 1734 struct mlxsw_sp_crif mock_crif = {}; 1735 1736 if (migrate_nhs) 1737 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 1738 1739 /* Plant a mock CRIF so that destroying the old RIF doesn't unoffload 1740 * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link. 1741 */ 1742 mlxsw_sp_crif_init(&mock_crif, crif->key.dev); 1743 old_rif->crif = &mock_crif; 1744 mock_crif.rif = old_rif; 1745 mlxsw_sp_rif_destroy(old_rif); 1746 } 1747 1748 static int 1749 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1750 struct mlxsw_sp_ipip_entry *ipip_entry, 1751 bool keep_encap, 1752 struct netlink_ext_ack *extack) 1753 { 1754 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1755 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1756 1757 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1758 ipip_entry->ipipt, 1759 ipip_entry->ol_dev, 1760 extack); 1761 if (IS_ERR(new_lb_rif)) 1762 return PTR_ERR(new_lb_rif); 1763 ipip_entry->ol_lb = new_lb_rif; 1764 1765 mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common, 1766 &new_lb_rif->common, keep_encap); 1767 return 0; 1768 } 1769 1770 /** 1771 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1772 * @mlxsw_sp: mlxsw_sp. 1773 * @ipip_entry: IPIP entry. 1774 * @recreate_loopback: Recreates the associated loopback RIF. 1775 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1776 * relevant when recreate_loopback is true. 1777 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1778 * is only relevant when recreate_loopback is false. 1779 * @extack: extack. 1780 * 1781 * Return: Non-zero value on failure. 1782 */ 1783 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1784 struct mlxsw_sp_ipip_entry *ipip_entry, 1785 bool recreate_loopback, 1786 bool keep_encap, 1787 bool update_nexthops, 1788 struct netlink_ext_ack *extack) 1789 { 1790 int err; 1791 1792 /* RIFs can't be edited, so to update loopback, we need to destroy and 1793 * recreate it. That creates a window of opportunity where RALUE and 1794 * RATR registers end up referencing a RIF that's already gone. RATRs 1795 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1796 * of RALUE, demote the decap route back. 1797 */ 1798 if (ipip_entry->decap_fib_entry) 1799 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1800 1801 if (recreate_loopback) { 1802 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1803 keep_encap, extack); 1804 if (err) 1805 return err; 1806 } else if (update_nexthops) { 1807 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1808 &ipip_entry->ol_lb->common); 1809 } 1810 1811 if (ipip_entry->ol_dev->flags & IFF_UP) 1812 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1813 1814 return 0; 1815 } 1816 1817 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1818 struct net_device *ol_dev, 1819 struct netlink_ext_ack *extack) 1820 { 1821 struct mlxsw_sp_ipip_entry *ipip_entry = 1822 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1823 1824 if (!ipip_entry) 1825 return 0; 1826 1827 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1828 true, false, false, extack); 1829 } 1830 1831 static int 1832 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1833 struct mlxsw_sp_ipip_entry *ipip_entry, 1834 struct net_device *ul_dev, 1835 bool *demote_this, 1836 struct netlink_ext_ack *extack) 1837 { 1838 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1839 enum mlxsw_sp_l3proto ul_proto; 1840 union mlxsw_sp_l3addr saddr; 1841 1842 /* Moving underlay to a different VRF might cause local address 1843 * conflict, and the conflicting tunnels need to be demoted. 1844 */ 1845 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1846 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1847 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1848 saddr, ul_tb_id, 1849 ipip_entry)) { 1850 *demote_this = true; 1851 return 0; 1852 } 1853 1854 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1855 true, true, false, extack); 1856 } 1857 1858 static int 1859 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1860 struct mlxsw_sp_ipip_entry *ipip_entry, 1861 struct net_device *ul_dev) 1862 { 1863 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1864 false, false, true, NULL); 1865 } 1866 1867 static int 1868 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1869 struct mlxsw_sp_ipip_entry *ipip_entry, 1870 struct net_device *ul_dev) 1871 { 1872 /* A down underlay device causes encapsulated packets to not be 1873 * forwarded, but decap still works. So refresh next hops without 1874 * touching anything else. 1875 */ 1876 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1877 false, false, true, NULL); 1878 } 1879 1880 static int 1881 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1882 struct net_device *ol_dev, 1883 struct netlink_ext_ack *extack) 1884 { 1885 const struct mlxsw_sp_ipip_ops *ipip_ops; 1886 struct mlxsw_sp_ipip_entry *ipip_entry; 1887 int err; 1888 1889 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1890 if (!ipip_entry) 1891 /* A change might make a tunnel eligible for offloading, but 1892 * that is currently not implemented. What falls to slow path 1893 * stays there. 1894 */ 1895 return 0; 1896 1897 /* A change might make a tunnel not eligible for offloading. */ 1898 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1899 ipip_entry->ipipt)) { 1900 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1901 return 0; 1902 } 1903 1904 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1905 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1906 return err; 1907 } 1908 1909 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1910 struct mlxsw_sp_ipip_entry *ipip_entry) 1911 { 1912 struct net_device *ol_dev = ipip_entry->ol_dev; 1913 1914 if (ol_dev->flags & IFF_UP) 1915 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1916 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1917 } 1918 1919 /* The configuration where several tunnels have the same local address in the 1920 * same underlay table needs special treatment in the HW. That is currently not 1921 * implemented in the driver. This function finds and demotes the first tunnel 1922 * with a given source address, except the one passed in the argument 1923 * `except'. 1924 */ 1925 bool 1926 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1927 enum mlxsw_sp_l3proto ul_proto, 1928 union mlxsw_sp_l3addr saddr, 1929 u32 ul_tb_id, 1930 const struct mlxsw_sp_ipip_entry *except) 1931 { 1932 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1933 1934 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1935 ipip_list_node) { 1936 if (ipip_entry != except && 1937 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1938 ul_tb_id, ipip_entry)) { 1939 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1940 return true; 1941 } 1942 } 1943 1944 return false; 1945 } 1946 1947 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1948 struct net_device *ul_dev) 1949 { 1950 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1951 1952 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1953 ipip_list_node) { 1954 struct net_device *ol_dev = ipip_entry->ol_dev; 1955 struct net_device *ipip_ul_dev; 1956 1957 rcu_read_lock(); 1958 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1959 rcu_read_unlock(); 1960 if (ipip_ul_dev == ul_dev) 1961 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1962 } 1963 } 1964 1965 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1966 struct net_device *ol_dev, 1967 unsigned long event, 1968 struct netdev_notifier_info *info) 1969 { 1970 struct netdev_notifier_changeupper_info *chup; 1971 struct netlink_ext_ack *extack; 1972 int err = 0; 1973 1974 switch (event) { 1975 case NETDEV_REGISTER: 1976 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1977 break; 1978 case NETDEV_UNREGISTER: 1979 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1980 break; 1981 case NETDEV_UP: 1982 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1983 break; 1984 case NETDEV_DOWN: 1985 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1986 break; 1987 case NETDEV_CHANGEUPPER: 1988 chup = container_of(info, typeof(*chup), info); 1989 extack = info->extack; 1990 if (netif_is_l3_master(chup->upper_dev)) 1991 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1992 ol_dev, 1993 extack); 1994 break; 1995 case NETDEV_CHANGE: 1996 extack = info->extack; 1997 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1998 ol_dev, extack); 1999 break; 2000 case NETDEV_CHANGEMTU: 2001 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 2002 break; 2003 } 2004 return err; 2005 } 2006 2007 static int 2008 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2009 struct mlxsw_sp_ipip_entry *ipip_entry, 2010 struct net_device *ul_dev, 2011 bool *demote_this, 2012 unsigned long event, 2013 struct netdev_notifier_info *info) 2014 { 2015 struct netdev_notifier_changeupper_info *chup; 2016 struct netlink_ext_ack *extack; 2017 2018 switch (event) { 2019 case NETDEV_CHANGEUPPER: 2020 chup = container_of(info, typeof(*chup), info); 2021 extack = info->extack; 2022 if (netif_is_l3_master(chup->upper_dev)) 2023 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 2024 ipip_entry, 2025 ul_dev, 2026 demote_this, 2027 extack); 2028 break; 2029 2030 case NETDEV_UP: 2031 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 2032 ul_dev); 2033 case NETDEV_DOWN: 2034 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 2035 ipip_entry, 2036 ul_dev); 2037 } 2038 return 0; 2039 } 2040 2041 static int 2042 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 2043 struct net_device *ul_dev, 2044 unsigned long event, 2045 struct netdev_notifier_info *info) 2046 { 2047 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 2048 int err; 2049 2050 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 2051 ul_dev, 2052 ipip_entry))) { 2053 struct mlxsw_sp_ipip_entry *prev; 2054 bool demote_this = false; 2055 2056 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 2057 ul_dev, &demote_this, 2058 event, info); 2059 if (err) { 2060 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 2061 ul_dev); 2062 return err; 2063 } 2064 2065 if (demote_this) { 2066 if (list_is_first(&ipip_entry->ipip_list_node, 2067 &mlxsw_sp->router->ipip_list)) 2068 prev = NULL; 2069 else 2070 /* This can't be cached from previous iteration, 2071 * because that entry could be gone now. 2072 */ 2073 prev = list_prev_entry(ipip_entry, 2074 ipip_list_node); 2075 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 2076 ipip_entry = prev; 2077 } 2078 } 2079 2080 return 0; 2081 } 2082 2083 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2084 enum mlxsw_sp_l3proto ul_proto, 2085 const union mlxsw_sp_l3addr *ul_sip, 2086 u32 tunnel_index) 2087 { 2088 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2089 struct mlxsw_sp_router *router = mlxsw_sp->router; 2090 struct mlxsw_sp_fib_entry *fib_entry; 2091 int err = 0; 2092 2093 mutex_lock(&mlxsw_sp->router->lock); 2094 2095 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 2096 err = -EINVAL; 2097 goto out; 2098 } 2099 2100 router->nve_decap_config.ul_tb_id = ul_tb_id; 2101 router->nve_decap_config.tunnel_index = tunnel_index; 2102 router->nve_decap_config.ul_proto = ul_proto; 2103 router->nve_decap_config.ul_sip = *ul_sip; 2104 router->nve_decap_config.valid = true; 2105 2106 /* It is valid to create a tunnel with a local IP and only later 2107 * assign this IP address to a local interface 2108 */ 2109 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2110 ul_proto, ul_sip, 2111 type); 2112 if (!fib_entry) 2113 goto out; 2114 2115 fib_entry->decap.tunnel_index = tunnel_index; 2116 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2117 2118 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2119 if (err) 2120 goto err_fib_entry_update; 2121 2122 goto out; 2123 2124 err_fib_entry_update: 2125 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2126 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2127 out: 2128 mutex_unlock(&mlxsw_sp->router->lock); 2129 return err; 2130 } 2131 2132 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2133 enum mlxsw_sp_l3proto ul_proto, 2134 const union mlxsw_sp_l3addr *ul_sip) 2135 { 2136 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2137 struct mlxsw_sp_router *router = mlxsw_sp->router; 2138 struct mlxsw_sp_fib_entry *fib_entry; 2139 2140 mutex_lock(&mlxsw_sp->router->lock); 2141 2142 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2143 goto out; 2144 2145 router->nve_decap_config.valid = false; 2146 2147 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2148 ul_proto, ul_sip, 2149 type); 2150 if (!fib_entry) 2151 goto out; 2152 2153 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2154 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2155 out: 2156 mutex_unlock(&mlxsw_sp->router->lock); 2157 } 2158 2159 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2160 u32 ul_tb_id, 2161 enum mlxsw_sp_l3proto ul_proto, 2162 const union mlxsw_sp_l3addr *ul_sip) 2163 { 2164 struct mlxsw_sp_router *router = mlxsw_sp->router; 2165 2166 return router->nve_decap_config.valid && 2167 router->nve_decap_config.ul_tb_id == ul_tb_id && 2168 router->nve_decap_config.ul_proto == ul_proto && 2169 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2170 sizeof(*ul_sip)); 2171 } 2172 2173 struct mlxsw_sp_neigh_key { 2174 struct neighbour *n; 2175 }; 2176 2177 struct mlxsw_sp_neigh_entry { 2178 struct list_head rif_list_node; 2179 struct rhash_head ht_node; 2180 struct mlxsw_sp_neigh_key key; 2181 u16 rif; 2182 bool connected; 2183 unsigned char ha[ETH_ALEN]; 2184 struct list_head nexthop_list; /* list of nexthops using 2185 * this neigh entry 2186 */ 2187 struct list_head nexthop_neighs_list_node; 2188 unsigned int counter_index; 2189 bool counter_valid; 2190 }; 2191 2192 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2193 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2194 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2195 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2196 }; 2197 2198 struct mlxsw_sp_neigh_entry * 2199 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2200 struct mlxsw_sp_neigh_entry *neigh_entry) 2201 { 2202 if (!neigh_entry) { 2203 if (list_empty(&rif->neigh_list)) 2204 return NULL; 2205 else 2206 return list_first_entry(&rif->neigh_list, 2207 typeof(*neigh_entry), 2208 rif_list_node); 2209 } 2210 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2211 return NULL; 2212 return list_next_entry(neigh_entry, rif_list_node); 2213 } 2214 2215 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2216 { 2217 return neigh_entry->key.n->tbl->family; 2218 } 2219 2220 unsigned char * 2221 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2222 { 2223 return neigh_entry->ha; 2224 } 2225 2226 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2227 { 2228 struct neighbour *n; 2229 2230 n = neigh_entry->key.n; 2231 return ntohl(*((__be32 *) n->primary_key)); 2232 } 2233 2234 struct in6_addr * 2235 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2236 { 2237 struct neighbour *n; 2238 2239 n = neigh_entry->key.n; 2240 return (struct in6_addr *) &n->primary_key; 2241 } 2242 2243 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2244 struct mlxsw_sp_neigh_entry *neigh_entry, 2245 u64 *p_counter) 2246 { 2247 if (!neigh_entry->counter_valid) 2248 return -EINVAL; 2249 2250 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2251 p_counter, NULL); 2252 } 2253 2254 static struct mlxsw_sp_neigh_entry * 2255 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2256 u16 rif) 2257 { 2258 struct mlxsw_sp_neigh_entry *neigh_entry; 2259 2260 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2261 if (!neigh_entry) 2262 return NULL; 2263 2264 neigh_entry->key.n = n; 2265 neigh_entry->rif = rif; 2266 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2267 2268 return neigh_entry; 2269 } 2270 2271 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2272 { 2273 kfree(neigh_entry); 2274 } 2275 2276 static int 2277 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2278 struct mlxsw_sp_neigh_entry *neigh_entry) 2279 { 2280 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2281 &neigh_entry->ht_node, 2282 mlxsw_sp_neigh_ht_params); 2283 } 2284 2285 static void 2286 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2287 struct mlxsw_sp_neigh_entry *neigh_entry) 2288 { 2289 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2290 &neigh_entry->ht_node, 2291 mlxsw_sp_neigh_ht_params); 2292 } 2293 2294 static bool 2295 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2296 struct mlxsw_sp_neigh_entry *neigh_entry) 2297 { 2298 struct devlink *devlink; 2299 const char *table_name; 2300 2301 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2302 case AF_INET: 2303 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2304 break; 2305 case AF_INET6: 2306 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2307 break; 2308 default: 2309 WARN_ON(1); 2310 return false; 2311 } 2312 2313 devlink = priv_to_devlink(mlxsw_sp->core); 2314 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2315 } 2316 2317 static void 2318 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2319 struct mlxsw_sp_neigh_entry *neigh_entry) 2320 { 2321 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2322 return; 2323 2324 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2325 return; 2326 2327 neigh_entry->counter_valid = true; 2328 } 2329 2330 static void 2331 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2332 struct mlxsw_sp_neigh_entry *neigh_entry) 2333 { 2334 if (!neigh_entry->counter_valid) 2335 return; 2336 mlxsw_sp_flow_counter_free(mlxsw_sp, 2337 neigh_entry->counter_index); 2338 neigh_entry->counter_valid = false; 2339 } 2340 2341 static struct mlxsw_sp_neigh_entry * 2342 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2343 { 2344 struct mlxsw_sp_neigh_entry *neigh_entry; 2345 struct mlxsw_sp_rif *rif; 2346 int err; 2347 2348 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2349 if (!rif) 2350 return ERR_PTR(-EINVAL); 2351 2352 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2353 if (!neigh_entry) 2354 return ERR_PTR(-ENOMEM); 2355 2356 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2357 if (err) 2358 goto err_neigh_entry_insert; 2359 2360 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2361 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2362 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2363 2364 return neigh_entry; 2365 2366 err_neigh_entry_insert: 2367 mlxsw_sp_neigh_entry_free(neigh_entry); 2368 return ERR_PTR(err); 2369 } 2370 2371 static void 2372 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2373 struct mlxsw_sp_neigh_entry *neigh_entry) 2374 { 2375 list_del(&neigh_entry->rif_list_node); 2376 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2377 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2378 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2379 mlxsw_sp_neigh_entry_free(neigh_entry); 2380 } 2381 2382 static struct mlxsw_sp_neigh_entry * 2383 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2384 { 2385 struct mlxsw_sp_neigh_key key; 2386 2387 key.n = n; 2388 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2389 &key, mlxsw_sp_neigh_ht_params); 2390 } 2391 2392 static void 2393 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2394 { 2395 unsigned long interval; 2396 2397 #if IS_ENABLED(CONFIG_IPV6) 2398 interval = min_t(unsigned long, 2399 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2400 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2401 #else 2402 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2403 #endif 2404 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2405 } 2406 2407 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2408 char *rauhtd_pl, 2409 int ent_index) 2410 { 2411 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2412 struct net_device *dev; 2413 struct neighbour *n; 2414 __be32 dipn; 2415 u32 dip; 2416 u16 rif; 2417 2418 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2419 2420 if (WARN_ON_ONCE(rif >= max_rifs)) 2421 return; 2422 if (!mlxsw_sp->router->rifs[rif]) { 2423 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2424 return; 2425 } 2426 2427 dipn = htonl(dip); 2428 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2429 n = neigh_lookup(&arp_tbl, &dipn, dev); 2430 if (!n) 2431 return; 2432 2433 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2434 neigh_event_send(n, NULL); 2435 neigh_release(n); 2436 } 2437 2438 #if IS_ENABLED(CONFIG_IPV6) 2439 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2440 char *rauhtd_pl, 2441 int rec_index) 2442 { 2443 struct net_device *dev; 2444 struct neighbour *n; 2445 struct in6_addr dip; 2446 u16 rif; 2447 2448 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2449 (char *) &dip); 2450 2451 if (!mlxsw_sp->router->rifs[rif]) { 2452 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2453 return; 2454 } 2455 2456 dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]); 2457 n = neigh_lookup(&nd_tbl, &dip, dev); 2458 if (!n) 2459 return; 2460 2461 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2462 neigh_event_send(n, NULL); 2463 neigh_release(n); 2464 } 2465 #else 2466 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2467 char *rauhtd_pl, 2468 int rec_index) 2469 { 2470 } 2471 #endif 2472 2473 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2474 char *rauhtd_pl, 2475 int rec_index) 2476 { 2477 u8 num_entries; 2478 int i; 2479 2480 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2481 rec_index); 2482 /* Hardware starts counting at 0, so add 1. */ 2483 num_entries++; 2484 2485 /* Each record consists of several neighbour entries. */ 2486 for (i = 0; i < num_entries; i++) { 2487 int ent_index; 2488 2489 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2490 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2491 ent_index); 2492 } 2493 2494 } 2495 2496 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2497 char *rauhtd_pl, 2498 int rec_index) 2499 { 2500 /* One record contains one entry. */ 2501 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2502 rec_index); 2503 } 2504 2505 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2506 char *rauhtd_pl, int rec_index) 2507 { 2508 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2509 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2510 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2511 rec_index); 2512 break; 2513 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2514 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2515 rec_index); 2516 break; 2517 } 2518 } 2519 2520 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2521 { 2522 u8 num_rec, last_rec_index, num_entries; 2523 2524 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2525 last_rec_index = num_rec - 1; 2526 2527 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2528 return false; 2529 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2530 MLXSW_REG_RAUHTD_TYPE_IPV6) 2531 return true; 2532 2533 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2534 last_rec_index); 2535 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2536 return true; 2537 return false; 2538 } 2539 2540 static int 2541 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2542 char *rauhtd_pl, 2543 enum mlxsw_reg_rauhtd_type type) 2544 { 2545 int i, num_rec; 2546 int err; 2547 2548 /* Ensure the RIF we read from the device does not change mid-dump. */ 2549 mutex_lock(&mlxsw_sp->router->lock); 2550 do { 2551 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2552 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2553 rauhtd_pl); 2554 if (err) { 2555 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2556 break; 2557 } 2558 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2559 for (i = 0; i < num_rec; i++) 2560 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2561 i); 2562 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2563 mutex_unlock(&mlxsw_sp->router->lock); 2564 2565 return err; 2566 } 2567 2568 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2569 { 2570 enum mlxsw_reg_rauhtd_type type; 2571 char *rauhtd_pl; 2572 int err; 2573 2574 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2575 return 0; 2576 2577 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2578 if (!rauhtd_pl) 2579 return -ENOMEM; 2580 2581 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2582 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2583 if (err) 2584 goto out; 2585 2586 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2587 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2588 out: 2589 kfree(rauhtd_pl); 2590 return err; 2591 } 2592 2593 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2594 { 2595 struct mlxsw_sp_neigh_entry *neigh_entry; 2596 2597 mutex_lock(&mlxsw_sp->router->lock); 2598 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2599 nexthop_neighs_list_node) 2600 /* If this neigh have nexthops, make the kernel think this neigh 2601 * is active regardless of the traffic. 2602 */ 2603 neigh_event_send(neigh_entry->key.n, NULL); 2604 mutex_unlock(&mlxsw_sp->router->lock); 2605 } 2606 2607 static void 2608 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2609 { 2610 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2611 2612 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2613 msecs_to_jiffies(interval)); 2614 } 2615 2616 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2617 { 2618 struct mlxsw_sp_router *router; 2619 int err; 2620 2621 router = container_of(work, struct mlxsw_sp_router, 2622 neighs_update.dw.work); 2623 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2624 if (err) 2625 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2626 2627 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2628 2629 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2630 } 2631 2632 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2633 { 2634 struct mlxsw_sp_neigh_entry *neigh_entry; 2635 struct mlxsw_sp_router *router; 2636 2637 router = container_of(work, struct mlxsw_sp_router, 2638 nexthop_probe_dw.work); 2639 /* Iterate over nexthop neighbours, find those who are unresolved and 2640 * send arp on them. This solves the chicken-egg problem when 2641 * the nexthop wouldn't get offloaded until the neighbor is resolved 2642 * but it wouldn't get resolved ever in case traffic is flowing in HW 2643 * using different nexthop. 2644 */ 2645 mutex_lock(&router->lock); 2646 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2647 nexthop_neighs_list_node) 2648 if (!neigh_entry->connected) 2649 neigh_event_send(neigh_entry->key.n, NULL); 2650 mutex_unlock(&router->lock); 2651 2652 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2653 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2654 } 2655 2656 static void 2657 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2658 struct mlxsw_sp_neigh_entry *neigh_entry, 2659 bool removing, bool dead); 2660 2661 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2662 { 2663 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2664 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2665 } 2666 2667 static int 2668 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2669 struct mlxsw_sp_neigh_entry *neigh_entry, 2670 enum mlxsw_reg_rauht_op op) 2671 { 2672 struct neighbour *n = neigh_entry->key.n; 2673 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2674 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2675 2676 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2677 dip); 2678 if (neigh_entry->counter_valid) 2679 mlxsw_reg_rauht_pack_counter(rauht_pl, 2680 neigh_entry->counter_index); 2681 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2682 } 2683 2684 static int 2685 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2686 struct mlxsw_sp_neigh_entry *neigh_entry, 2687 enum mlxsw_reg_rauht_op op) 2688 { 2689 struct neighbour *n = neigh_entry->key.n; 2690 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2691 const char *dip = n->primary_key; 2692 2693 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2694 dip); 2695 if (neigh_entry->counter_valid) 2696 mlxsw_reg_rauht_pack_counter(rauht_pl, 2697 neigh_entry->counter_index); 2698 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2699 } 2700 2701 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2702 { 2703 struct neighbour *n = neigh_entry->key.n; 2704 2705 /* Packets with a link-local destination address are trapped 2706 * after LPM lookup and never reach the neighbour table, so 2707 * there is no need to program such neighbours to the device. 2708 */ 2709 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2710 IPV6_ADDR_LINKLOCAL) 2711 return true; 2712 return false; 2713 } 2714 2715 static void 2716 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2717 struct mlxsw_sp_neigh_entry *neigh_entry, 2718 bool adding) 2719 { 2720 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2721 int err; 2722 2723 if (!adding && !neigh_entry->connected) 2724 return; 2725 neigh_entry->connected = adding; 2726 if (neigh_entry->key.n->tbl->family == AF_INET) { 2727 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2728 op); 2729 if (err) 2730 return; 2731 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2732 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2733 return; 2734 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2735 op); 2736 if (err) 2737 return; 2738 } else { 2739 WARN_ON_ONCE(1); 2740 return; 2741 } 2742 2743 if (adding) 2744 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2745 else 2746 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2747 } 2748 2749 void 2750 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2751 struct mlxsw_sp_neigh_entry *neigh_entry, 2752 bool adding) 2753 { 2754 if (adding) 2755 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2756 else 2757 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2758 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2759 } 2760 2761 struct mlxsw_sp_netevent_work { 2762 struct work_struct work; 2763 struct mlxsw_sp *mlxsw_sp; 2764 struct neighbour *n; 2765 }; 2766 2767 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2768 { 2769 struct mlxsw_sp_netevent_work *net_work = 2770 container_of(work, struct mlxsw_sp_netevent_work, work); 2771 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2772 struct mlxsw_sp_neigh_entry *neigh_entry; 2773 struct neighbour *n = net_work->n; 2774 unsigned char ha[ETH_ALEN]; 2775 bool entry_connected; 2776 u8 nud_state, dead; 2777 2778 /* If these parameters are changed after we release the lock, 2779 * then we are guaranteed to receive another event letting us 2780 * know about it. 2781 */ 2782 read_lock_bh(&n->lock); 2783 memcpy(ha, n->ha, ETH_ALEN); 2784 nud_state = n->nud_state; 2785 dead = n->dead; 2786 read_unlock_bh(&n->lock); 2787 2788 mutex_lock(&mlxsw_sp->router->lock); 2789 mlxsw_sp_span_respin(mlxsw_sp); 2790 2791 entry_connected = nud_state & NUD_VALID && !dead; 2792 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2793 if (!entry_connected && !neigh_entry) 2794 goto out; 2795 if (!neigh_entry) { 2796 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2797 if (IS_ERR(neigh_entry)) 2798 goto out; 2799 } 2800 2801 if (neigh_entry->connected && entry_connected && 2802 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2803 goto out; 2804 2805 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2806 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2807 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2808 dead); 2809 2810 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2811 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2812 2813 out: 2814 mutex_unlock(&mlxsw_sp->router->lock); 2815 neigh_release(n); 2816 kfree(net_work); 2817 } 2818 2819 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2820 2821 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2822 { 2823 struct mlxsw_sp_netevent_work *net_work = 2824 container_of(work, struct mlxsw_sp_netevent_work, work); 2825 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2826 2827 mlxsw_sp_mp_hash_init(mlxsw_sp); 2828 kfree(net_work); 2829 } 2830 2831 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2832 2833 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2834 { 2835 struct mlxsw_sp_netevent_work *net_work = 2836 container_of(work, struct mlxsw_sp_netevent_work, work); 2837 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2838 2839 __mlxsw_sp_router_init(mlxsw_sp); 2840 kfree(net_work); 2841 } 2842 2843 static int mlxsw_sp_router_schedule_work(struct net *net, 2844 struct mlxsw_sp_router *router, 2845 struct neighbour *n, 2846 void (*cb)(struct work_struct *)) 2847 { 2848 struct mlxsw_sp_netevent_work *net_work; 2849 2850 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2851 return NOTIFY_DONE; 2852 2853 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2854 if (!net_work) 2855 return NOTIFY_BAD; 2856 2857 INIT_WORK(&net_work->work, cb); 2858 net_work->mlxsw_sp = router->mlxsw_sp; 2859 net_work->n = n; 2860 mlxsw_core_schedule_work(&net_work->work); 2861 return NOTIFY_DONE; 2862 } 2863 2864 static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev) 2865 { 2866 struct mlxsw_sp_port *mlxsw_sp_port; 2867 2868 rcu_read_lock(); 2869 mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev); 2870 rcu_read_unlock(); 2871 return !!mlxsw_sp_port; 2872 } 2873 2874 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2875 unsigned long event, void *ptr) 2876 { 2877 struct mlxsw_sp_router *router; 2878 unsigned long interval; 2879 struct neigh_parms *p; 2880 struct neighbour *n; 2881 struct net *net; 2882 2883 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2884 2885 switch (event) { 2886 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2887 p = ptr; 2888 2889 /* We don't care about changes in the default table. */ 2890 if (!p->dev || (p->tbl->family != AF_INET && 2891 p->tbl->family != AF_INET6)) 2892 return NOTIFY_DONE; 2893 2894 /* We are in atomic context and can't take RTNL mutex, 2895 * so use RCU variant to walk the device chain. 2896 */ 2897 if (!mlxsw_sp_dev_lower_is_port(p->dev)) 2898 return NOTIFY_DONE; 2899 2900 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2901 router->neighs_update.interval = interval; 2902 break; 2903 case NETEVENT_NEIGH_UPDATE: 2904 n = ptr; 2905 net = neigh_parms_net(n->parms); 2906 2907 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2908 return NOTIFY_DONE; 2909 2910 if (!mlxsw_sp_dev_lower_is_port(n->dev)) 2911 return NOTIFY_DONE; 2912 2913 /* Take a reference to ensure the neighbour won't be 2914 * destructed until we drop the reference in delayed 2915 * work. 2916 */ 2917 neigh_clone(n); 2918 return mlxsw_sp_router_schedule_work(net, router, n, 2919 mlxsw_sp_router_neigh_event_work); 2920 2921 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2922 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2923 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2924 mlxsw_sp_router_mp_hash_event_work); 2925 2926 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2927 return mlxsw_sp_router_schedule_work(ptr, router, NULL, 2928 mlxsw_sp_router_update_priority_work); 2929 } 2930 2931 return NOTIFY_DONE; 2932 } 2933 2934 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2935 { 2936 int err; 2937 2938 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2939 &mlxsw_sp_neigh_ht_params); 2940 if (err) 2941 return err; 2942 2943 /* Initialize the polling interval according to the default 2944 * table. 2945 */ 2946 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2947 2948 /* Create the delayed works for the activity_update */ 2949 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2950 mlxsw_sp_router_neighs_update_work); 2951 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2952 mlxsw_sp_router_probe_unresolved_nexthops); 2953 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2954 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2955 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2956 return 0; 2957 } 2958 2959 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2960 { 2961 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2962 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2963 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2964 } 2965 2966 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2967 struct mlxsw_sp_rif *rif) 2968 { 2969 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2970 2971 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2972 rif_list_node) { 2973 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2974 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2975 } 2976 } 2977 2978 enum mlxsw_sp_nexthop_type { 2979 MLXSW_SP_NEXTHOP_TYPE_ETH, 2980 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2981 }; 2982 2983 enum mlxsw_sp_nexthop_action { 2984 /* Nexthop forwards packets to an egress RIF */ 2985 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2986 /* Nexthop discards packets */ 2987 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2988 /* Nexthop traps packets */ 2989 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2990 }; 2991 2992 struct mlxsw_sp_nexthop_key { 2993 struct fib_nh *fib_nh; 2994 }; 2995 2996 struct mlxsw_sp_nexthop { 2997 struct list_head neigh_list_node; /* member of neigh entry list */ 2998 struct list_head crif_list_node; 2999 struct list_head router_list_node; 3000 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 3001 * this nexthop belongs to 3002 */ 3003 struct rhash_head ht_node; 3004 struct neigh_table *neigh_tbl; 3005 struct mlxsw_sp_nexthop_key key; 3006 unsigned char gw_addr[sizeof(struct in6_addr)]; 3007 int ifindex; 3008 int nh_weight; 3009 int norm_nh_weight; 3010 int num_adj_entries; 3011 struct mlxsw_sp_crif *crif; 3012 u8 should_offload:1, /* set indicates this nexthop should be written 3013 * to the adjacency table. 3014 */ 3015 offloaded:1, /* set indicates this nexthop was written to the 3016 * adjacency table. 3017 */ 3018 update:1; /* set indicates this nexthop should be updated in the 3019 * adjacency table (f.e., its MAC changed). 3020 */ 3021 enum mlxsw_sp_nexthop_action action; 3022 enum mlxsw_sp_nexthop_type type; 3023 union { 3024 struct mlxsw_sp_neigh_entry *neigh_entry; 3025 struct mlxsw_sp_ipip_entry *ipip_entry; 3026 }; 3027 unsigned int counter_index; 3028 bool counter_valid; 3029 }; 3030 3031 static struct net_device * 3032 mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh) 3033 { 3034 if (!nh->crif) 3035 return NULL; 3036 return nh->crif->key.dev; 3037 } 3038 3039 enum mlxsw_sp_nexthop_group_type { 3040 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 3041 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 3042 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 3043 }; 3044 3045 struct mlxsw_sp_nexthop_group_info { 3046 struct mlxsw_sp_nexthop_group *nh_grp; 3047 u32 adj_index; 3048 u16 ecmp_size; 3049 u16 count; 3050 int sum_norm_weight; 3051 u8 adj_index_valid:1, 3052 gateway:1, /* routes using the group use a gateway */ 3053 is_resilient:1; 3054 struct list_head list; /* member in nh_res_grp_list */ 3055 struct mlxsw_sp_nexthop nexthops[]; 3056 }; 3057 3058 static struct mlxsw_sp_rif * 3059 mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi) 3060 { 3061 struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif; 3062 3063 if (!crif) 3064 return NULL; 3065 return crif->rif; 3066 } 3067 3068 struct mlxsw_sp_nexthop_group_vr_key { 3069 u16 vr_id; 3070 enum mlxsw_sp_l3proto proto; 3071 }; 3072 3073 struct mlxsw_sp_nexthop_group_vr_entry { 3074 struct list_head list; /* member in vr_list */ 3075 struct rhash_head ht_node; /* member in vr_ht */ 3076 refcount_t ref_count; 3077 struct mlxsw_sp_nexthop_group_vr_key key; 3078 }; 3079 3080 struct mlxsw_sp_nexthop_group { 3081 struct rhash_head ht_node; 3082 struct list_head fib_list; /* list of fib entries that use this group */ 3083 union { 3084 struct { 3085 struct fib_info *fi; 3086 } ipv4; 3087 struct { 3088 u32 id; 3089 } obj; 3090 }; 3091 struct mlxsw_sp_nexthop_group_info *nhgi; 3092 struct list_head vr_list; 3093 struct rhashtable vr_ht; 3094 enum mlxsw_sp_nexthop_group_type type; 3095 bool can_destroy; 3096 }; 3097 3098 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 3099 struct mlxsw_sp_nexthop *nh) 3100 { 3101 struct devlink *devlink; 3102 3103 devlink = priv_to_devlink(mlxsw_sp->core); 3104 if (!devlink_dpipe_table_counter_enabled(devlink, 3105 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 3106 return; 3107 3108 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 3109 return; 3110 3111 nh->counter_valid = true; 3112 } 3113 3114 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3115 struct mlxsw_sp_nexthop *nh) 3116 { 3117 if (!nh->counter_valid) 3118 return; 3119 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3120 nh->counter_valid = false; 3121 } 3122 3123 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3124 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3125 { 3126 if (!nh->counter_valid) 3127 return -EINVAL; 3128 3129 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3130 p_counter, NULL); 3131 } 3132 3133 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3134 struct mlxsw_sp_nexthop *nh) 3135 { 3136 if (!nh) { 3137 if (list_empty(&router->nexthop_list)) 3138 return NULL; 3139 else 3140 return list_first_entry(&router->nexthop_list, 3141 typeof(*nh), router_list_node); 3142 } 3143 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3144 return NULL; 3145 return list_next_entry(nh, router_list_node); 3146 } 3147 3148 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3149 { 3150 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3151 } 3152 3153 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3154 { 3155 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3156 !mlxsw_sp_nexthop_is_forward(nh)) 3157 return NULL; 3158 return nh->neigh_entry->ha; 3159 } 3160 3161 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3162 u32 *p_adj_size, u32 *p_adj_hash_index) 3163 { 3164 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3165 u32 adj_hash_index = 0; 3166 int i; 3167 3168 if (!nh->offloaded || !nhgi->adj_index_valid) 3169 return -EINVAL; 3170 3171 *p_adj_index = nhgi->adj_index; 3172 *p_adj_size = nhgi->ecmp_size; 3173 3174 for (i = 0; i < nhgi->count; i++) { 3175 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3176 3177 if (nh_iter == nh) 3178 break; 3179 if (nh_iter->offloaded) 3180 adj_hash_index += nh_iter->num_adj_entries; 3181 } 3182 3183 *p_adj_hash_index = adj_hash_index; 3184 return 0; 3185 } 3186 3187 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3188 { 3189 if (WARN_ON(!nh->crif)) 3190 return NULL; 3191 return nh->crif->rif; 3192 } 3193 3194 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3195 { 3196 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3197 int i; 3198 3199 for (i = 0; i < nhgi->count; i++) { 3200 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3201 3202 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3203 return true; 3204 } 3205 return false; 3206 } 3207 3208 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3209 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3210 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3211 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3212 .automatic_shrinking = true, 3213 }; 3214 3215 static struct mlxsw_sp_nexthop_group_vr_entry * 3216 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3217 const struct mlxsw_sp_fib *fib) 3218 { 3219 struct mlxsw_sp_nexthop_group_vr_key key; 3220 3221 memset(&key, 0, sizeof(key)); 3222 key.vr_id = fib->vr->id; 3223 key.proto = fib->proto; 3224 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3225 mlxsw_sp_nexthop_group_vr_ht_params); 3226 } 3227 3228 static int 3229 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3230 const struct mlxsw_sp_fib *fib) 3231 { 3232 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3233 int err; 3234 3235 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3236 if (!vr_entry) 3237 return -ENOMEM; 3238 3239 vr_entry->key.vr_id = fib->vr->id; 3240 vr_entry->key.proto = fib->proto; 3241 refcount_set(&vr_entry->ref_count, 1); 3242 3243 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3244 mlxsw_sp_nexthop_group_vr_ht_params); 3245 if (err) 3246 goto err_hashtable_insert; 3247 3248 list_add(&vr_entry->list, &nh_grp->vr_list); 3249 3250 return 0; 3251 3252 err_hashtable_insert: 3253 kfree(vr_entry); 3254 return err; 3255 } 3256 3257 static void 3258 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3259 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3260 { 3261 list_del(&vr_entry->list); 3262 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3263 mlxsw_sp_nexthop_group_vr_ht_params); 3264 kfree(vr_entry); 3265 } 3266 3267 static int 3268 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3269 const struct mlxsw_sp_fib *fib) 3270 { 3271 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3272 3273 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3274 if (vr_entry) { 3275 refcount_inc(&vr_entry->ref_count); 3276 return 0; 3277 } 3278 3279 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3280 } 3281 3282 static void 3283 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3284 const struct mlxsw_sp_fib *fib) 3285 { 3286 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3287 3288 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3289 if (WARN_ON_ONCE(!vr_entry)) 3290 return; 3291 3292 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3293 return; 3294 3295 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3296 } 3297 3298 struct mlxsw_sp_nexthop_group_cmp_arg { 3299 enum mlxsw_sp_nexthop_group_type type; 3300 union { 3301 struct fib_info *fi; 3302 struct mlxsw_sp_fib6_entry *fib6_entry; 3303 u32 id; 3304 }; 3305 }; 3306 3307 static bool 3308 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3309 const struct in6_addr *gw, int ifindex, 3310 int weight) 3311 { 3312 int i; 3313 3314 for (i = 0; i < nh_grp->nhgi->count; i++) { 3315 const struct mlxsw_sp_nexthop *nh; 3316 3317 nh = &nh_grp->nhgi->nexthops[i]; 3318 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3319 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3320 return true; 3321 } 3322 3323 return false; 3324 } 3325 3326 static bool 3327 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3328 const struct mlxsw_sp_fib6_entry *fib6_entry) 3329 { 3330 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3331 3332 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3333 return false; 3334 3335 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3336 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3337 struct in6_addr *gw; 3338 int ifindex, weight; 3339 3340 ifindex = fib6_nh->fib_nh_dev->ifindex; 3341 weight = fib6_nh->fib_nh_weight; 3342 gw = &fib6_nh->fib_nh_gw6; 3343 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3344 weight)) 3345 return false; 3346 } 3347 3348 return true; 3349 } 3350 3351 static int 3352 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3353 { 3354 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3355 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3356 3357 if (nh_grp->type != cmp_arg->type) 3358 return 1; 3359 3360 switch (cmp_arg->type) { 3361 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3362 return cmp_arg->fi != nh_grp->ipv4.fi; 3363 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3364 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3365 cmp_arg->fib6_entry); 3366 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3367 return cmp_arg->id != nh_grp->obj.id; 3368 default: 3369 WARN_ON(1); 3370 return 1; 3371 } 3372 } 3373 3374 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3375 { 3376 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3377 const struct mlxsw_sp_nexthop *nh; 3378 struct fib_info *fi; 3379 unsigned int val; 3380 int i; 3381 3382 switch (nh_grp->type) { 3383 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3384 fi = nh_grp->ipv4.fi; 3385 return jhash(&fi, sizeof(fi), seed); 3386 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3387 val = nh_grp->nhgi->count; 3388 for (i = 0; i < nh_grp->nhgi->count; i++) { 3389 nh = &nh_grp->nhgi->nexthops[i]; 3390 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3391 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3392 } 3393 return jhash(&val, sizeof(val), seed); 3394 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3395 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3396 default: 3397 WARN_ON(1); 3398 return 0; 3399 } 3400 } 3401 3402 static u32 3403 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3404 { 3405 unsigned int val = fib6_entry->nrt6; 3406 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3407 3408 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3409 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3410 struct net_device *dev = fib6_nh->fib_nh_dev; 3411 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3412 3413 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3414 val ^= jhash(gw, sizeof(*gw), seed); 3415 } 3416 3417 return jhash(&val, sizeof(val), seed); 3418 } 3419 3420 static u32 3421 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3422 { 3423 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3424 3425 switch (cmp_arg->type) { 3426 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3427 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3428 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3429 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3430 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3431 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3432 default: 3433 WARN_ON(1); 3434 return 0; 3435 } 3436 } 3437 3438 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3439 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3440 .hashfn = mlxsw_sp_nexthop_group_hash, 3441 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3442 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3443 }; 3444 3445 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3446 struct mlxsw_sp_nexthop_group *nh_grp) 3447 { 3448 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3449 !nh_grp->nhgi->gateway) 3450 return 0; 3451 3452 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3453 &nh_grp->ht_node, 3454 mlxsw_sp_nexthop_group_ht_params); 3455 } 3456 3457 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3458 struct mlxsw_sp_nexthop_group *nh_grp) 3459 { 3460 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3461 !nh_grp->nhgi->gateway) 3462 return; 3463 3464 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3465 &nh_grp->ht_node, 3466 mlxsw_sp_nexthop_group_ht_params); 3467 } 3468 3469 static struct mlxsw_sp_nexthop_group * 3470 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3471 struct fib_info *fi) 3472 { 3473 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3474 3475 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3476 cmp_arg.fi = fi; 3477 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3478 &cmp_arg, 3479 mlxsw_sp_nexthop_group_ht_params); 3480 } 3481 3482 static struct mlxsw_sp_nexthop_group * 3483 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3484 struct mlxsw_sp_fib6_entry *fib6_entry) 3485 { 3486 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3487 3488 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3489 cmp_arg.fib6_entry = fib6_entry; 3490 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3491 &cmp_arg, 3492 mlxsw_sp_nexthop_group_ht_params); 3493 } 3494 3495 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3496 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3497 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3498 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3499 }; 3500 3501 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3502 struct mlxsw_sp_nexthop *nh) 3503 { 3504 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3505 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3506 } 3507 3508 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3509 struct mlxsw_sp_nexthop *nh) 3510 { 3511 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3512 mlxsw_sp_nexthop_ht_params); 3513 } 3514 3515 static struct mlxsw_sp_nexthop * 3516 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3517 struct mlxsw_sp_nexthop_key key) 3518 { 3519 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3520 mlxsw_sp_nexthop_ht_params); 3521 } 3522 3523 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3524 enum mlxsw_sp_l3proto proto, 3525 u16 vr_id, 3526 u32 adj_index, u16 ecmp_size, 3527 u32 new_adj_index, 3528 u16 new_ecmp_size) 3529 { 3530 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3531 3532 mlxsw_reg_raleu_pack(raleu_pl, 3533 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3534 adj_index, ecmp_size, new_adj_index, 3535 new_ecmp_size); 3536 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3537 } 3538 3539 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3540 struct mlxsw_sp_nexthop_group *nh_grp, 3541 u32 old_adj_index, u16 old_ecmp_size) 3542 { 3543 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3544 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3545 int err; 3546 3547 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3548 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3549 vr_entry->key.proto, 3550 vr_entry->key.vr_id, 3551 old_adj_index, 3552 old_ecmp_size, 3553 nhgi->adj_index, 3554 nhgi->ecmp_size); 3555 if (err) 3556 goto err_mass_update_vr; 3557 } 3558 return 0; 3559 3560 err_mass_update_vr: 3561 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3562 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3563 vr_entry->key.vr_id, 3564 nhgi->adj_index, 3565 nhgi->ecmp_size, 3566 old_adj_index, old_ecmp_size); 3567 return err; 3568 } 3569 3570 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3571 u32 adj_index, 3572 struct mlxsw_sp_nexthop *nh, 3573 bool force, char *ratr_pl) 3574 { 3575 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3576 struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh); 3577 enum mlxsw_reg_ratr_op op; 3578 u16 rif_index; 3579 3580 rif_index = rif ? rif->rif_index : 3581 mlxsw_sp->router->lb_crif->rif->rif_index; 3582 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3583 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3584 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3585 adj_index, rif_index); 3586 switch (nh->action) { 3587 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3588 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3589 break; 3590 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3591 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3592 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3593 break; 3594 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3595 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3596 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3597 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3598 break; 3599 default: 3600 WARN_ON_ONCE(1); 3601 return -EINVAL; 3602 } 3603 if (nh->counter_valid) 3604 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3605 else 3606 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3607 3608 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3609 } 3610 3611 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3612 struct mlxsw_sp_nexthop *nh, bool force, 3613 char *ratr_pl) 3614 { 3615 int i; 3616 3617 for (i = 0; i < nh->num_adj_entries; i++) { 3618 int err; 3619 3620 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3621 nh, force, ratr_pl); 3622 if (err) 3623 return err; 3624 } 3625 3626 return 0; 3627 } 3628 3629 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3630 u32 adj_index, 3631 struct mlxsw_sp_nexthop *nh, 3632 bool force, char *ratr_pl) 3633 { 3634 const struct mlxsw_sp_ipip_ops *ipip_ops; 3635 3636 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3637 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3638 force, ratr_pl); 3639 } 3640 3641 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3642 u32 adj_index, 3643 struct mlxsw_sp_nexthop *nh, bool force, 3644 char *ratr_pl) 3645 { 3646 int i; 3647 3648 for (i = 0; i < nh->num_adj_entries; i++) { 3649 int err; 3650 3651 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3652 nh, force, ratr_pl); 3653 if (err) 3654 return err; 3655 } 3656 3657 return 0; 3658 } 3659 3660 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3661 struct mlxsw_sp_nexthop *nh, bool force, 3662 char *ratr_pl) 3663 { 3664 /* When action is discard or trap, the nexthop must be 3665 * programmed as an Ethernet nexthop. 3666 */ 3667 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3668 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3669 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3670 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3671 force, ratr_pl); 3672 else 3673 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3674 force, ratr_pl); 3675 } 3676 3677 static int 3678 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3679 struct mlxsw_sp_nexthop_group_info *nhgi, 3680 bool reallocate) 3681 { 3682 char ratr_pl[MLXSW_REG_RATR_LEN]; 3683 u32 adj_index = nhgi->adj_index; /* base */ 3684 struct mlxsw_sp_nexthop *nh; 3685 int i; 3686 3687 for (i = 0; i < nhgi->count; i++) { 3688 nh = &nhgi->nexthops[i]; 3689 3690 if (!nh->should_offload) { 3691 nh->offloaded = 0; 3692 continue; 3693 } 3694 3695 if (nh->update || reallocate) { 3696 int err = 0; 3697 3698 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3699 true, ratr_pl); 3700 if (err) 3701 return err; 3702 nh->update = 0; 3703 nh->offloaded = 1; 3704 } 3705 adj_index += nh->num_adj_entries; 3706 } 3707 return 0; 3708 } 3709 3710 static int 3711 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3712 struct mlxsw_sp_nexthop_group *nh_grp) 3713 { 3714 struct mlxsw_sp_fib_entry *fib_entry; 3715 int err; 3716 3717 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3718 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3719 if (err) 3720 return err; 3721 } 3722 return 0; 3723 } 3724 3725 struct mlxsw_sp_adj_grp_size_range { 3726 u16 start; /* Inclusive */ 3727 u16 end; /* Inclusive */ 3728 }; 3729 3730 /* Ordered by range start value */ 3731 static const struct mlxsw_sp_adj_grp_size_range 3732 mlxsw_sp1_adj_grp_size_ranges[] = { 3733 { .start = 1, .end = 64 }, 3734 { .start = 512, .end = 512 }, 3735 { .start = 1024, .end = 1024 }, 3736 { .start = 2048, .end = 2048 }, 3737 { .start = 4096, .end = 4096 }, 3738 }; 3739 3740 /* Ordered by range start value */ 3741 static const struct mlxsw_sp_adj_grp_size_range 3742 mlxsw_sp2_adj_grp_size_ranges[] = { 3743 { .start = 1, .end = 128 }, 3744 { .start = 256, .end = 256 }, 3745 { .start = 512, .end = 512 }, 3746 { .start = 1024, .end = 1024 }, 3747 { .start = 2048, .end = 2048 }, 3748 { .start = 4096, .end = 4096 }, 3749 }; 3750 3751 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3752 u16 *p_adj_grp_size) 3753 { 3754 int i; 3755 3756 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3757 const struct mlxsw_sp_adj_grp_size_range *size_range; 3758 3759 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3760 3761 if (*p_adj_grp_size >= size_range->start && 3762 *p_adj_grp_size <= size_range->end) 3763 return; 3764 3765 if (*p_adj_grp_size <= size_range->end) { 3766 *p_adj_grp_size = size_range->end; 3767 return; 3768 } 3769 } 3770 } 3771 3772 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3773 u16 *p_adj_grp_size, 3774 unsigned int alloc_size) 3775 { 3776 int i; 3777 3778 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3779 const struct mlxsw_sp_adj_grp_size_range *size_range; 3780 3781 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3782 3783 if (alloc_size >= size_range->end) { 3784 *p_adj_grp_size = size_range->end; 3785 return; 3786 } 3787 } 3788 } 3789 3790 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3791 u16 *p_adj_grp_size) 3792 { 3793 unsigned int alloc_size; 3794 int err; 3795 3796 /* Round up the requested group size to the next size supported 3797 * by the device and make sure the request can be satisfied. 3798 */ 3799 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3800 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3801 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3802 *p_adj_grp_size, &alloc_size); 3803 if (err) 3804 return err; 3805 /* It is possible the allocation results in more allocated 3806 * entries than requested. Try to use as much of them as 3807 * possible. 3808 */ 3809 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3810 3811 return 0; 3812 } 3813 3814 static void 3815 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3816 { 3817 int i, g = 0, sum_norm_weight = 0; 3818 struct mlxsw_sp_nexthop *nh; 3819 3820 for (i = 0; i < nhgi->count; i++) { 3821 nh = &nhgi->nexthops[i]; 3822 3823 if (!nh->should_offload) 3824 continue; 3825 if (g > 0) 3826 g = gcd(nh->nh_weight, g); 3827 else 3828 g = nh->nh_weight; 3829 } 3830 3831 for (i = 0; i < nhgi->count; i++) { 3832 nh = &nhgi->nexthops[i]; 3833 3834 if (!nh->should_offload) 3835 continue; 3836 nh->norm_nh_weight = nh->nh_weight / g; 3837 sum_norm_weight += nh->norm_nh_weight; 3838 } 3839 3840 nhgi->sum_norm_weight = sum_norm_weight; 3841 } 3842 3843 static void 3844 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3845 { 3846 int i, weight = 0, lower_bound = 0; 3847 int total = nhgi->sum_norm_weight; 3848 u16 ecmp_size = nhgi->ecmp_size; 3849 3850 for (i = 0; i < nhgi->count; i++) { 3851 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3852 int upper_bound; 3853 3854 if (!nh->should_offload) 3855 continue; 3856 weight += nh->norm_nh_weight; 3857 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3858 nh->num_adj_entries = upper_bound - lower_bound; 3859 lower_bound = upper_bound; 3860 } 3861 } 3862 3863 static struct mlxsw_sp_nexthop * 3864 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3865 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3866 3867 static void 3868 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3869 struct mlxsw_sp_nexthop_group *nh_grp) 3870 { 3871 int i; 3872 3873 for (i = 0; i < nh_grp->nhgi->count; i++) { 3874 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3875 3876 if (nh->offloaded) 3877 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3878 else 3879 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3880 } 3881 } 3882 3883 static void 3884 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3885 struct mlxsw_sp_fib6_entry *fib6_entry) 3886 { 3887 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3888 3889 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3890 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3891 struct mlxsw_sp_nexthop *nh; 3892 3893 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3894 if (nh && nh->offloaded) 3895 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3896 else 3897 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3898 } 3899 } 3900 3901 static void 3902 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3903 struct mlxsw_sp_nexthop_group *nh_grp) 3904 { 3905 struct mlxsw_sp_fib6_entry *fib6_entry; 3906 3907 /* Unfortunately, in IPv6 the route and the nexthop are described by 3908 * the same struct, so we need to iterate over all the routes using the 3909 * nexthop group and set / clear the offload indication for them. 3910 */ 3911 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3912 common.nexthop_group_node) 3913 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3914 } 3915 3916 static void 3917 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3918 const struct mlxsw_sp_nexthop *nh, 3919 u16 bucket_index) 3920 { 3921 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3922 bool offload = false, trap = false; 3923 3924 if (nh->offloaded) { 3925 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3926 trap = true; 3927 else 3928 offload = true; 3929 } 3930 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3931 bucket_index, offload, trap); 3932 } 3933 3934 static void 3935 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3936 struct mlxsw_sp_nexthop_group *nh_grp) 3937 { 3938 int i; 3939 3940 /* Do not update the flags if the nexthop group is being destroyed 3941 * since: 3942 * 1. The nexthop objects is being deleted, in which case the flags are 3943 * irrelevant. 3944 * 2. The nexthop group was replaced by a newer group, in which case 3945 * the flags of the nexthop object were already updated based on the 3946 * new group. 3947 */ 3948 if (nh_grp->can_destroy) 3949 return; 3950 3951 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3952 nh_grp->nhgi->adj_index_valid, false); 3953 3954 /* Update flags of individual nexthop buckets in case of a resilient 3955 * nexthop group. 3956 */ 3957 if (!nh_grp->nhgi->is_resilient) 3958 return; 3959 3960 for (i = 0; i < nh_grp->nhgi->count; i++) { 3961 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3962 3963 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3964 } 3965 } 3966 3967 static void 3968 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3969 struct mlxsw_sp_nexthop_group *nh_grp) 3970 { 3971 switch (nh_grp->type) { 3972 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3973 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3974 break; 3975 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3976 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3977 break; 3978 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3979 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3980 break; 3981 } 3982 } 3983 3984 static int 3985 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3986 struct mlxsw_sp_nexthop_group *nh_grp) 3987 { 3988 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3989 u16 ecmp_size, old_ecmp_size; 3990 struct mlxsw_sp_nexthop *nh; 3991 bool offload_change = false; 3992 u32 adj_index; 3993 bool old_adj_index_valid; 3994 u32 old_adj_index; 3995 int i, err2, err; 3996 3997 if (!nhgi->gateway) 3998 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3999 4000 for (i = 0; i < nhgi->count; i++) { 4001 nh = &nhgi->nexthops[i]; 4002 4003 if (nh->should_offload != nh->offloaded) { 4004 offload_change = true; 4005 if (nh->should_offload) 4006 nh->update = 1; 4007 } 4008 } 4009 if (!offload_change) { 4010 /* Nothing was added or removed, so no need to reallocate. Just 4011 * update MAC on existing adjacency indexes. 4012 */ 4013 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 4014 if (err) { 4015 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4016 goto set_trap; 4017 } 4018 /* Flags of individual nexthop buckets might need to be 4019 * updated. 4020 */ 4021 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4022 return 0; 4023 } 4024 mlxsw_sp_nexthop_group_normalize(nhgi); 4025 if (!nhgi->sum_norm_weight) { 4026 /* No neigh of this group is connected so we just set 4027 * the trap and let everthing flow through kernel. 4028 */ 4029 err = 0; 4030 goto set_trap; 4031 } 4032 4033 ecmp_size = nhgi->sum_norm_weight; 4034 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 4035 if (err) 4036 /* No valid allocation size available. */ 4037 goto set_trap; 4038 4039 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4040 ecmp_size, &adj_index); 4041 if (err) { 4042 /* We ran out of KVD linear space, just set the 4043 * trap and let everything flow through kernel. 4044 */ 4045 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 4046 goto set_trap; 4047 } 4048 old_adj_index_valid = nhgi->adj_index_valid; 4049 old_adj_index = nhgi->adj_index; 4050 old_ecmp_size = nhgi->ecmp_size; 4051 nhgi->adj_index_valid = 1; 4052 nhgi->adj_index = adj_index; 4053 nhgi->ecmp_size = ecmp_size; 4054 mlxsw_sp_nexthop_group_rebalance(nhgi); 4055 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 4056 if (err) { 4057 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 4058 goto set_trap; 4059 } 4060 4061 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4062 4063 if (!old_adj_index_valid) { 4064 /* The trap was set for fib entries, so we have to call 4065 * fib entry update to unset it and use adjacency index. 4066 */ 4067 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4068 if (err) { 4069 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 4070 goto set_trap; 4071 } 4072 return 0; 4073 } 4074 4075 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 4076 old_adj_index, old_ecmp_size); 4077 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4078 old_ecmp_size, old_adj_index); 4079 if (err) { 4080 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 4081 goto set_trap; 4082 } 4083 4084 return 0; 4085 4086 set_trap: 4087 old_adj_index_valid = nhgi->adj_index_valid; 4088 nhgi->adj_index_valid = 0; 4089 for (i = 0; i < nhgi->count; i++) { 4090 nh = &nhgi->nexthops[i]; 4091 nh->offloaded = 0; 4092 } 4093 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 4094 if (err2) 4095 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 4096 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 4097 if (old_adj_index_valid) 4098 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4099 nhgi->ecmp_size, nhgi->adj_index); 4100 return err; 4101 } 4102 4103 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 4104 bool removing) 4105 { 4106 if (!removing) { 4107 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 4108 nh->should_offload = 1; 4109 } else if (nh->nhgi->is_resilient) { 4110 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4111 nh->should_offload = 1; 4112 } else { 4113 nh->should_offload = 0; 4114 } 4115 nh->update = 1; 4116 } 4117 4118 static int 4119 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4120 struct mlxsw_sp_neigh_entry *neigh_entry) 4121 { 4122 struct neighbour *n, *old_n = neigh_entry->key.n; 4123 struct mlxsw_sp_nexthop *nh; 4124 struct net_device *dev; 4125 bool entry_connected; 4126 u8 nud_state, dead; 4127 int err; 4128 4129 nh = list_first_entry(&neigh_entry->nexthop_list, 4130 struct mlxsw_sp_nexthop, neigh_list_node); 4131 dev = mlxsw_sp_nexthop_dev(nh); 4132 4133 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4134 if (!n) { 4135 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4136 if (IS_ERR(n)) 4137 return PTR_ERR(n); 4138 neigh_event_send(n, NULL); 4139 } 4140 4141 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4142 neigh_entry->key.n = n; 4143 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4144 if (err) 4145 goto err_neigh_entry_insert; 4146 4147 read_lock_bh(&n->lock); 4148 nud_state = n->nud_state; 4149 dead = n->dead; 4150 read_unlock_bh(&n->lock); 4151 entry_connected = nud_state & NUD_VALID && !dead; 4152 4153 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4154 neigh_list_node) { 4155 neigh_release(old_n); 4156 neigh_clone(n); 4157 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4158 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4159 } 4160 4161 neigh_release(n); 4162 4163 return 0; 4164 4165 err_neigh_entry_insert: 4166 neigh_entry->key.n = old_n; 4167 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4168 neigh_release(n); 4169 return err; 4170 } 4171 4172 static void 4173 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4174 struct mlxsw_sp_neigh_entry *neigh_entry, 4175 bool removing, bool dead) 4176 { 4177 struct mlxsw_sp_nexthop *nh; 4178 4179 if (list_empty(&neigh_entry->nexthop_list)) 4180 return; 4181 4182 if (dead) { 4183 int err; 4184 4185 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4186 neigh_entry); 4187 if (err) 4188 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4189 return; 4190 } 4191 4192 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4193 neigh_list_node) { 4194 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4195 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4196 } 4197 } 4198 4199 static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh, 4200 struct mlxsw_sp_crif *crif) 4201 { 4202 if (nh->crif) 4203 return; 4204 4205 nh->crif = crif; 4206 list_add(&nh->crif_list_node, &crif->nexthop_list); 4207 } 4208 4209 static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh) 4210 { 4211 if (!nh->crif) 4212 return; 4213 4214 list_del(&nh->crif_list_node); 4215 nh->crif = NULL; 4216 } 4217 4218 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4219 struct mlxsw_sp_nexthop *nh) 4220 { 4221 struct mlxsw_sp_neigh_entry *neigh_entry; 4222 struct net_device *dev; 4223 struct neighbour *n; 4224 u8 nud_state, dead; 4225 int err; 4226 4227 if (WARN_ON(!nh->crif->rif)) 4228 return 0; 4229 4230 if (!nh->nhgi->gateway || nh->neigh_entry) 4231 return 0; 4232 dev = mlxsw_sp_nexthop_dev(nh); 4233 4234 /* Take a reference of neigh here ensuring that neigh would 4235 * not be destructed before the nexthop entry is finished. 4236 * The reference is taken either in neigh_lookup() or 4237 * in neigh_create() in case n is not found. 4238 */ 4239 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev); 4240 if (!n) { 4241 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev); 4242 if (IS_ERR(n)) 4243 return PTR_ERR(n); 4244 neigh_event_send(n, NULL); 4245 } 4246 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4247 if (!neigh_entry) { 4248 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4249 if (IS_ERR(neigh_entry)) { 4250 err = -EINVAL; 4251 goto err_neigh_entry_create; 4252 } 4253 } 4254 4255 /* If that is the first nexthop connected to that neigh, add to 4256 * nexthop_neighs_list 4257 */ 4258 if (list_empty(&neigh_entry->nexthop_list)) 4259 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4260 &mlxsw_sp->router->nexthop_neighs_list); 4261 4262 nh->neigh_entry = neigh_entry; 4263 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4264 read_lock_bh(&n->lock); 4265 nud_state = n->nud_state; 4266 dead = n->dead; 4267 read_unlock_bh(&n->lock); 4268 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4269 4270 return 0; 4271 4272 err_neigh_entry_create: 4273 neigh_release(n); 4274 return err; 4275 } 4276 4277 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4278 struct mlxsw_sp_nexthop *nh) 4279 { 4280 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4281 struct neighbour *n; 4282 4283 if (!neigh_entry) 4284 return; 4285 n = neigh_entry->key.n; 4286 4287 __mlxsw_sp_nexthop_neigh_update(nh, true); 4288 list_del(&nh->neigh_list_node); 4289 nh->neigh_entry = NULL; 4290 4291 /* If that is the last nexthop connected to that neigh, remove from 4292 * nexthop_neighs_list 4293 */ 4294 if (list_empty(&neigh_entry->nexthop_list)) 4295 list_del(&neigh_entry->nexthop_neighs_list_node); 4296 4297 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4298 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4299 4300 neigh_release(n); 4301 } 4302 4303 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4304 { 4305 struct net_device *ul_dev; 4306 bool is_up; 4307 4308 rcu_read_lock(); 4309 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4310 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4311 rcu_read_unlock(); 4312 4313 return is_up; 4314 } 4315 4316 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4317 struct mlxsw_sp_nexthop *nh, 4318 struct mlxsw_sp_ipip_entry *ipip_entry) 4319 { 4320 struct mlxsw_sp_crif *crif; 4321 bool removing; 4322 4323 if (!nh->nhgi->gateway || nh->ipip_entry) 4324 return; 4325 4326 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev); 4327 if (WARN_ON(!crif)) 4328 return; 4329 4330 nh->ipip_entry = ipip_entry; 4331 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4332 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4333 mlxsw_sp_nexthop_crif_init(nh, crif); 4334 } 4335 4336 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4337 struct mlxsw_sp_nexthop *nh) 4338 { 4339 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4340 4341 if (!ipip_entry) 4342 return; 4343 4344 __mlxsw_sp_nexthop_neigh_update(nh, true); 4345 nh->ipip_entry = NULL; 4346 } 4347 4348 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4349 const struct fib_nh *fib_nh, 4350 enum mlxsw_sp_ipip_type *p_ipipt) 4351 { 4352 struct net_device *dev = fib_nh->fib_nh_dev; 4353 4354 return dev && 4355 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4356 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4357 } 4358 4359 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4360 struct mlxsw_sp_nexthop *nh, 4361 const struct net_device *dev) 4362 { 4363 const struct mlxsw_sp_ipip_ops *ipip_ops; 4364 struct mlxsw_sp_ipip_entry *ipip_entry; 4365 struct mlxsw_sp_crif *crif; 4366 int err; 4367 4368 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4369 if (ipip_entry) { 4370 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4371 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4372 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4373 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4374 return 0; 4375 } 4376 } 4377 4378 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4379 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev); 4380 if (!crif) 4381 return 0; 4382 4383 mlxsw_sp_nexthop_crif_init(nh, crif); 4384 4385 if (!crif->rif) 4386 return 0; 4387 4388 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4389 if (err) 4390 goto err_neigh_init; 4391 4392 return 0; 4393 4394 err_neigh_init: 4395 mlxsw_sp_nexthop_crif_fini(nh); 4396 return err; 4397 } 4398 4399 static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp, 4400 struct mlxsw_sp_nexthop *nh) 4401 { 4402 switch (nh->type) { 4403 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4404 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4405 break; 4406 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4407 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4408 break; 4409 } 4410 } 4411 4412 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4413 struct mlxsw_sp_nexthop *nh) 4414 { 4415 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4416 mlxsw_sp_nexthop_crif_fini(nh); 4417 } 4418 4419 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4420 struct mlxsw_sp_nexthop_group *nh_grp, 4421 struct mlxsw_sp_nexthop *nh, 4422 struct fib_nh *fib_nh) 4423 { 4424 struct net_device *dev = fib_nh->fib_nh_dev; 4425 struct in_device *in_dev; 4426 int err; 4427 4428 nh->nhgi = nh_grp->nhgi; 4429 nh->key.fib_nh = fib_nh; 4430 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4431 nh->nh_weight = fib_nh->fib_nh_weight; 4432 #else 4433 nh->nh_weight = 1; 4434 #endif 4435 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4436 nh->neigh_tbl = &arp_tbl; 4437 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4438 if (err) 4439 return err; 4440 4441 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4442 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4443 4444 if (!dev) 4445 return 0; 4446 nh->ifindex = dev->ifindex; 4447 4448 rcu_read_lock(); 4449 in_dev = __in_dev_get_rcu(dev); 4450 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4451 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4452 rcu_read_unlock(); 4453 return 0; 4454 } 4455 rcu_read_unlock(); 4456 4457 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4458 if (err) 4459 goto err_nexthop_neigh_init; 4460 4461 return 0; 4462 4463 err_nexthop_neigh_init: 4464 list_del(&nh->router_list_node); 4465 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4466 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4467 return err; 4468 } 4469 4470 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4471 struct mlxsw_sp_nexthop *nh) 4472 { 4473 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4474 list_del(&nh->router_list_node); 4475 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4476 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4477 } 4478 4479 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4480 unsigned long event, struct fib_nh *fib_nh) 4481 { 4482 struct mlxsw_sp_nexthop_key key; 4483 struct mlxsw_sp_nexthop *nh; 4484 4485 key.fib_nh = fib_nh; 4486 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4487 if (!nh) 4488 return; 4489 4490 switch (event) { 4491 case FIB_EVENT_NH_ADD: 4492 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4493 break; 4494 case FIB_EVENT_NH_DEL: 4495 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4496 break; 4497 } 4498 4499 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4500 } 4501 4502 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4503 struct mlxsw_sp_rif *rif) 4504 { 4505 struct net_device *dev = mlxsw_sp_rif_dev(rif); 4506 struct mlxsw_sp_nexthop *nh; 4507 bool removing; 4508 4509 list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) { 4510 switch (nh->type) { 4511 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4512 removing = false; 4513 break; 4514 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4515 removing = !mlxsw_sp_ipip_netdev_ul_up(dev); 4516 break; 4517 default: 4518 WARN_ON(1); 4519 continue; 4520 } 4521 4522 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4523 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4524 } 4525 } 4526 4527 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4528 struct mlxsw_sp_rif *rif) 4529 { 4530 struct mlxsw_sp_nexthop *nh, *tmp; 4531 4532 list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list, 4533 crif_list_node) { 4534 mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh); 4535 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4536 } 4537 } 4538 4539 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4540 { 4541 enum mlxsw_reg_ratr_trap_action trap_action; 4542 char ratr_pl[MLXSW_REG_RATR_LEN]; 4543 int err; 4544 4545 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4546 &mlxsw_sp->router->adj_trap_index); 4547 if (err) 4548 return err; 4549 4550 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4551 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4552 MLXSW_REG_RATR_TYPE_ETHERNET, 4553 mlxsw_sp->router->adj_trap_index, 4554 mlxsw_sp->router->lb_crif->rif->rif_index); 4555 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4556 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4557 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4558 if (err) 4559 goto err_ratr_write; 4560 4561 return 0; 4562 4563 err_ratr_write: 4564 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4565 mlxsw_sp->router->adj_trap_index); 4566 return err; 4567 } 4568 4569 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4570 { 4571 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4572 mlxsw_sp->router->adj_trap_index); 4573 } 4574 4575 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4576 { 4577 int err; 4578 4579 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4580 return 0; 4581 4582 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4583 if (err) 4584 return err; 4585 4586 refcount_set(&mlxsw_sp->router->num_groups, 1); 4587 4588 return 0; 4589 } 4590 4591 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4592 { 4593 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4594 return; 4595 4596 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4597 } 4598 4599 static void 4600 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4601 const struct mlxsw_sp_nexthop_group *nh_grp, 4602 unsigned long *activity) 4603 { 4604 char *ratrad_pl; 4605 int i, err; 4606 4607 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4608 if (!ratrad_pl) 4609 return; 4610 4611 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4612 nh_grp->nhgi->count); 4613 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4614 if (err) 4615 goto out; 4616 4617 for (i = 0; i < nh_grp->nhgi->count; i++) { 4618 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4619 continue; 4620 bitmap_set(activity, i, 1); 4621 } 4622 4623 out: 4624 kfree(ratrad_pl); 4625 } 4626 4627 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4628 4629 static void 4630 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4631 const struct mlxsw_sp_nexthop_group *nh_grp) 4632 { 4633 unsigned long *activity; 4634 4635 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4636 if (!activity) 4637 return; 4638 4639 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4640 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4641 nh_grp->nhgi->count, activity); 4642 4643 bitmap_free(activity); 4644 } 4645 4646 static void 4647 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4648 { 4649 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4650 4651 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4652 msecs_to_jiffies(interval)); 4653 } 4654 4655 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4656 { 4657 struct mlxsw_sp_nexthop_group_info *nhgi; 4658 struct mlxsw_sp_router *router; 4659 bool reschedule = false; 4660 4661 router = container_of(work, struct mlxsw_sp_router, 4662 nh_grp_activity_dw.work); 4663 4664 mutex_lock(&router->lock); 4665 4666 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4667 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4668 reschedule = true; 4669 } 4670 4671 mutex_unlock(&router->lock); 4672 4673 if (!reschedule) 4674 return; 4675 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4676 } 4677 4678 static int 4679 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4680 const struct nh_notifier_single_info *nh, 4681 struct netlink_ext_ack *extack) 4682 { 4683 int err = -EINVAL; 4684 4685 if (nh->is_fdb) 4686 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4687 else if (nh->has_encap) 4688 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4689 else 4690 err = 0; 4691 4692 return err; 4693 } 4694 4695 static int 4696 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4697 const struct nh_notifier_single_info *nh, 4698 struct netlink_ext_ack *extack) 4699 { 4700 int err; 4701 4702 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4703 if (err) 4704 return err; 4705 4706 /* Device only nexthops with an IPIP device are programmed as 4707 * encapsulating adjacency entries. 4708 */ 4709 if (!nh->gw_family && !nh->is_reject && 4710 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4711 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4712 return -EINVAL; 4713 } 4714 4715 return 0; 4716 } 4717 4718 static int 4719 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4720 const struct nh_notifier_grp_info *nh_grp, 4721 struct netlink_ext_ack *extack) 4722 { 4723 int i; 4724 4725 if (nh_grp->is_fdb) { 4726 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4727 return -EINVAL; 4728 } 4729 4730 for (i = 0; i < nh_grp->num_nh; i++) { 4731 const struct nh_notifier_single_info *nh; 4732 int err; 4733 4734 nh = &nh_grp->nh_entries[i].nh; 4735 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4736 extack); 4737 if (err) 4738 return err; 4739 } 4740 4741 return 0; 4742 } 4743 4744 static int 4745 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4746 const struct nh_notifier_res_table_info *nh_res_table, 4747 struct netlink_ext_ack *extack) 4748 { 4749 unsigned int alloc_size; 4750 bool valid_size = false; 4751 int err, i; 4752 4753 if (nh_res_table->num_nh_buckets < 32) { 4754 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4755 return -EINVAL; 4756 } 4757 4758 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4759 const struct mlxsw_sp_adj_grp_size_range *size_range; 4760 4761 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4762 4763 if (nh_res_table->num_nh_buckets >= size_range->start && 4764 nh_res_table->num_nh_buckets <= size_range->end) { 4765 valid_size = true; 4766 break; 4767 } 4768 } 4769 4770 if (!valid_size) { 4771 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4772 return -EINVAL; 4773 } 4774 4775 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4776 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4777 nh_res_table->num_nh_buckets, 4778 &alloc_size); 4779 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4780 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4781 return -EINVAL; 4782 } 4783 4784 return 0; 4785 } 4786 4787 static int 4788 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4789 const struct nh_notifier_res_table_info *nh_res_table, 4790 struct netlink_ext_ack *extack) 4791 { 4792 int err; 4793 u16 i; 4794 4795 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4796 nh_res_table, 4797 extack); 4798 if (err) 4799 return err; 4800 4801 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4802 const struct nh_notifier_single_info *nh; 4803 int err; 4804 4805 nh = &nh_res_table->nhs[i]; 4806 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4807 extack); 4808 if (err) 4809 return err; 4810 } 4811 4812 return 0; 4813 } 4814 4815 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4816 unsigned long event, 4817 struct nh_notifier_info *info) 4818 { 4819 struct nh_notifier_single_info *nh; 4820 4821 if (event != NEXTHOP_EVENT_REPLACE && 4822 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4823 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4824 return 0; 4825 4826 switch (info->type) { 4827 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4828 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4829 info->extack); 4830 case NH_NOTIFIER_INFO_TYPE_GRP: 4831 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4832 info->nh_grp, 4833 info->extack); 4834 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4835 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4836 info->nh_res_table, 4837 info->extack); 4838 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4839 nh = &info->nh_res_bucket->new_nh; 4840 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4841 info->extack); 4842 default: 4843 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4844 return -EOPNOTSUPP; 4845 } 4846 } 4847 4848 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4849 const struct nh_notifier_info *info) 4850 { 4851 const struct net_device *dev; 4852 4853 switch (info->type) { 4854 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4855 dev = info->nh->dev; 4856 return info->nh->gw_family || info->nh->is_reject || 4857 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4858 case NH_NOTIFIER_INFO_TYPE_GRP: 4859 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4860 /* Already validated earlier. */ 4861 return true; 4862 default: 4863 return false; 4864 } 4865 } 4866 4867 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4868 struct mlxsw_sp_nexthop *nh) 4869 { 4870 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4871 nh->should_offload = 1; 4872 /* While nexthops that discard packets do not forward packets 4873 * via an egress RIF, they still need to be programmed using a 4874 * valid RIF, so use the loopback RIF created during init. 4875 */ 4876 nh->crif = mlxsw_sp->router->lb_crif; 4877 } 4878 4879 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4880 struct mlxsw_sp_nexthop *nh) 4881 { 4882 nh->crif = NULL; 4883 nh->should_offload = 0; 4884 } 4885 4886 static int 4887 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4888 struct mlxsw_sp_nexthop_group *nh_grp, 4889 struct mlxsw_sp_nexthop *nh, 4890 struct nh_notifier_single_info *nh_obj, int weight) 4891 { 4892 struct net_device *dev = nh_obj->dev; 4893 int err; 4894 4895 nh->nhgi = nh_grp->nhgi; 4896 nh->nh_weight = weight; 4897 4898 switch (nh_obj->gw_family) { 4899 case AF_INET: 4900 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4901 nh->neigh_tbl = &arp_tbl; 4902 break; 4903 case AF_INET6: 4904 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4905 #if IS_ENABLED(CONFIG_IPV6) 4906 nh->neigh_tbl = &nd_tbl; 4907 #endif 4908 break; 4909 } 4910 4911 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4912 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4913 nh->ifindex = dev->ifindex; 4914 4915 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4916 if (err) 4917 goto err_type_init; 4918 4919 if (nh_obj->is_reject) 4920 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4921 4922 /* In a resilient nexthop group, all the nexthops must be written to 4923 * the adjacency table. Even if they do not have a valid neighbour or 4924 * RIF. 4925 */ 4926 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4927 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4928 nh->should_offload = 1; 4929 } 4930 4931 return 0; 4932 4933 err_type_init: 4934 list_del(&nh->router_list_node); 4935 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4936 return err; 4937 } 4938 4939 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4940 struct mlxsw_sp_nexthop *nh) 4941 { 4942 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4943 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4944 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4945 list_del(&nh->router_list_node); 4946 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4947 nh->should_offload = 0; 4948 } 4949 4950 static int 4951 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4952 struct mlxsw_sp_nexthop_group *nh_grp, 4953 struct nh_notifier_info *info) 4954 { 4955 struct mlxsw_sp_nexthop_group_info *nhgi; 4956 struct mlxsw_sp_nexthop *nh; 4957 bool is_resilient = false; 4958 unsigned int nhs; 4959 int err, i; 4960 4961 switch (info->type) { 4962 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4963 nhs = 1; 4964 break; 4965 case NH_NOTIFIER_INFO_TYPE_GRP: 4966 nhs = info->nh_grp->num_nh; 4967 break; 4968 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4969 nhs = info->nh_res_table->num_nh_buckets; 4970 is_resilient = true; 4971 break; 4972 default: 4973 return -EINVAL; 4974 } 4975 4976 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4977 if (!nhgi) 4978 return -ENOMEM; 4979 nh_grp->nhgi = nhgi; 4980 nhgi->nh_grp = nh_grp; 4981 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4982 nhgi->is_resilient = is_resilient; 4983 nhgi->count = nhs; 4984 for (i = 0; i < nhgi->count; i++) { 4985 struct nh_notifier_single_info *nh_obj; 4986 int weight; 4987 4988 nh = &nhgi->nexthops[i]; 4989 switch (info->type) { 4990 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4991 nh_obj = info->nh; 4992 weight = 1; 4993 break; 4994 case NH_NOTIFIER_INFO_TYPE_GRP: 4995 nh_obj = &info->nh_grp->nh_entries[i].nh; 4996 weight = info->nh_grp->nh_entries[i].weight; 4997 break; 4998 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4999 nh_obj = &info->nh_res_table->nhs[i]; 5000 weight = 1; 5001 break; 5002 default: 5003 err = -EINVAL; 5004 goto err_nexthop_obj_init; 5005 } 5006 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 5007 weight); 5008 if (err) 5009 goto err_nexthop_obj_init; 5010 } 5011 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5012 if (err) 5013 goto err_group_inc; 5014 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5015 if (err) { 5016 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 5017 goto err_group_refresh; 5018 } 5019 5020 /* Add resilient nexthop groups to a list so that the activity of their 5021 * nexthop buckets will be periodically queried and cleared. 5022 */ 5023 if (nhgi->is_resilient) { 5024 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5025 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 5026 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 5027 } 5028 5029 return 0; 5030 5031 err_group_refresh: 5032 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5033 err_group_inc: 5034 i = nhgi->count; 5035 err_nexthop_obj_init: 5036 for (i--; i >= 0; i--) { 5037 nh = &nhgi->nexthops[i]; 5038 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5039 } 5040 kfree(nhgi); 5041 return err; 5042 } 5043 5044 static void 5045 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5046 struct mlxsw_sp_nexthop_group *nh_grp) 5047 { 5048 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5049 struct mlxsw_sp_router *router = mlxsw_sp->router; 5050 int i; 5051 5052 if (nhgi->is_resilient) { 5053 list_del(&nhgi->list); 5054 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 5055 cancel_delayed_work(&router->nh_grp_activity_dw); 5056 } 5057 5058 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5059 for (i = nhgi->count - 1; i >= 0; i--) { 5060 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5061 5062 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5063 } 5064 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5065 WARN_ON_ONCE(nhgi->adj_index_valid); 5066 kfree(nhgi); 5067 } 5068 5069 static struct mlxsw_sp_nexthop_group * 5070 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 5071 struct nh_notifier_info *info) 5072 { 5073 struct mlxsw_sp_nexthop_group *nh_grp; 5074 int err; 5075 5076 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5077 if (!nh_grp) 5078 return ERR_PTR(-ENOMEM); 5079 INIT_LIST_HEAD(&nh_grp->vr_list); 5080 err = rhashtable_init(&nh_grp->vr_ht, 5081 &mlxsw_sp_nexthop_group_vr_ht_params); 5082 if (err) 5083 goto err_nexthop_group_vr_ht_init; 5084 INIT_LIST_HEAD(&nh_grp->fib_list); 5085 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5086 nh_grp->obj.id = info->id; 5087 5088 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 5089 if (err) 5090 goto err_nexthop_group_info_init; 5091 5092 nh_grp->can_destroy = false; 5093 5094 return nh_grp; 5095 5096 err_nexthop_group_info_init: 5097 rhashtable_destroy(&nh_grp->vr_ht); 5098 err_nexthop_group_vr_ht_init: 5099 kfree(nh_grp); 5100 return ERR_PTR(err); 5101 } 5102 5103 static void 5104 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 5105 struct mlxsw_sp_nexthop_group *nh_grp) 5106 { 5107 if (!nh_grp->can_destroy) 5108 return; 5109 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 5110 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 5111 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5112 rhashtable_destroy(&nh_grp->vr_ht); 5113 kfree(nh_grp); 5114 } 5115 5116 static struct mlxsw_sp_nexthop_group * 5117 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 5118 { 5119 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 5120 5121 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 5122 cmp_arg.id = id; 5123 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 5124 &cmp_arg, 5125 mlxsw_sp_nexthop_group_ht_params); 5126 } 5127 5128 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5129 struct mlxsw_sp_nexthop_group *nh_grp) 5130 { 5131 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5132 } 5133 5134 static int 5135 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5136 struct mlxsw_sp_nexthop_group *nh_grp, 5137 struct mlxsw_sp_nexthop_group *old_nh_grp, 5138 struct netlink_ext_ack *extack) 5139 { 5140 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5141 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5142 int err; 5143 5144 old_nh_grp->nhgi = new_nhgi; 5145 new_nhgi->nh_grp = old_nh_grp; 5146 nh_grp->nhgi = old_nhgi; 5147 old_nhgi->nh_grp = nh_grp; 5148 5149 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5150 /* Both the old adjacency index and the new one are valid. 5151 * Routes are currently using the old one. Tell the device to 5152 * replace the old adjacency index with the new one. 5153 */ 5154 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5155 old_nhgi->adj_index, 5156 old_nhgi->ecmp_size); 5157 if (err) { 5158 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5159 goto err_out; 5160 } 5161 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5162 /* The old adjacency index is valid, while the new one is not. 5163 * Iterate over all the routes using the group and change them 5164 * to trap packets to the CPU. 5165 */ 5166 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5167 if (err) { 5168 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5169 goto err_out; 5170 } 5171 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5172 /* The old adjacency index is invalid, while the new one is. 5173 * Iterate over all the routes using the group and change them 5174 * to forward packets using the new valid index. 5175 */ 5176 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5177 if (err) { 5178 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5179 goto err_out; 5180 } 5181 } 5182 5183 /* Make sure the flags are set / cleared based on the new nexthop group 5184 * information. 5185 */ 5186 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5187 5188 /* At this point 'nh_grp' is just a shell that is not used by anyone 5189 * and its nexthop group info is the old info that was just replaced 5190 * with the new one. Remove it. 5191 */ 5192 nh_grp->can_destroy = true; 5193 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5194 5195 return 0; 5196 5197 err_out: 5198 old_nhgi->nh_grp = old_nh_grp; 5199 nh_grp->nhgi = new_nhgi; 5200 new_nhgi->nh_grp = nh_grp; 5201 old_nh_grp->nhgi = old_nhgi; 5202 return err; 5203 } 5204 5205 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5206 struct nh_notifier_info *info) 5207 { 5208 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5209 struct netlink_ext_ack *extack = info->extack; 5210 int err; 5211 5212 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5213 if (IS_ERR(nh_grp)) 5214 return PTR_ERR(nh_grp); 5215 5216 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5217 if (!old_nh_grp) 5218 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5219 else 5220 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5221 old_nh_grp, extack); 5222 5223 if (err) { 5224 nh_grp->can_destroy = true; 5225 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5226 } 5227 5228 return err; 5229 } 5230 5231 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5232 struct nh_notifier_info *info) 5233 { 5234 struct mlxsw_sp_nexthop_group *nh_grp; 5235 5236 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5237 if (!nh_grp) 5238 return; 5239 5240 nh_grp->can_destroy = true; 5241 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5242 5243 /* If the group still has routes using it, then defer the delete 5244 * operation until the last route using it is deleted. 5245 */ 5246 if (!list_empty(&nh_grp->fib_list)) 5247 return; 5248 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5249 } 5250 5251 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5252 u32 adj_index, char *ratr_pl) 5253 { 5254 MLXSW_REG_ZERO(ratr, ratr_pl); 5255 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5256 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5257 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5258 5259 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5260 } 5261 5262 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5263 { 5264 /* Clear the opcode and activity on both the old and new payload as 5265 * they are irrelevant for the comparison. 5266 */ 5267 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5268 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5269 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5270 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5271 5272 /* If the contents of the adjacency entry are consistent with the 5273 * replacement request, then replacement was successful. 5274 */ 5275 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5276 return 0; 5277 5278 return -EINVAL; 5279 } 5280 5281 static int 5282 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5283 struct mlxsw_sp_nexthop *nh, 5284 struct nh_notifier_info *info) 5285 { 5286 u16 bucket_index = info->nh_res_bucket->bucket_index; 5287 struct netlink_ext_ack *extack = info->extack; 5288 bool force = info->nh_res_bucket->force; 5289 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5290 char ratr_pl[MLXSW_REG_RATR_LEN]; 5291 u32 adj_index; 5292 int err; 5293 5294 /* No point in trying an atomic replacement if the idle timer interval 5295 * is smaller than the interval in which we query and clear activity. 5296 */ 5297 if (!force && info->nh_res_bucket->idle_timer_ms < 5298 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5299 force = true; 5300 5301 adj_index = nh->nhgi->adj_index + bucket_index; 5302 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5303 if (err) { 5304 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5305 return err; 5306 } 5307 5308 if (!force) { 5309 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5310 ratr_pl_new); 5311 if (err) { 5312 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5313 return err; 5314 } 5315 5316 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5317 if (err) { 5318 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5319 return err; 5320 } 5321 } 5322 5323 nh->update = 0; 5324 nh->offloaded = 1; 5325 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5326 5327 return 0; 5328 } 5329 5330 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5331 struct nh_notifier_info *info) 5332 { 5333 u16 bucket_index = info->nh_res_bucket->bucket_index; 5334 struct netlink_ext_ack *extack = info->extack; 5335 struct mlxsw_sp_nexthop_group_info *nhgi; 5336 struct nh_notifier_single_info *nh_obj; 5337 struct mlxsw_sp_nexthop_group *nh_grp; 5338 struct mlxsw_sp_nexthop *nh; 5339 int err; 5340 5341 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5342 if (!nh_grp) { 5343 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5344 return -EINVAL; 5345 } 5346 5347 nhgi = nh_grp->nhgi; 5348 5349 if (bucket_index >= nhgi->count) { 5350 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5351 return -EINVAL; 5352 } 5353 5354 nh = &nhgi->nexthops[bucket_index]; 5355 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5356 5357 nh_obj = &info->nh_res_bucket->new_nh; 5358 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5359 if (err) { 5360 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5361 goto err_nexthop_obj_init; 5362 } 5363 5364 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5365 if (err) 5366 goto err_nexthop_obj_bucket_adj_update; 5367 5368 return 0; 5369 5370 err_nexthop_obj_bucket_adj_update: 5371 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5372 err_nexthop_obj_init: 5373 nh_obj = &info->nh_res_bucket->old_nh; 5374 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5375 /* The old adjacency entry was not overwritten */ 5376 nh->update = 0; 5377 nh->offloaded = 1; 5378 return err; 5379 } 5380 5381 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5382 unsigned long event, void *ptr) 5383 { 5384 struct nh_notifier_info *info = ptr; 5385 struct mlxsw_sp_router *router; 5386 int err = 0; 5387 5388 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5389 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5390 if (err) 5391 goto out; 5392 5393 mutex_lock(&router->lock); 5394 5395 switch (event) { 5396 case NEXTHOP_EVENT_REPLACE: 5397 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5398 break; 5399 case NEXTHOP_EVENT_DEL: 5400 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5401 break; 5402 case NEXTHOP_EVENT_BUCKET_REPLACE: 5403 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5404 info); 5405 break; 5406 default: 5407 break; 5408 } 5409 5410 mutex_unlock(&router->lock); 5411 5412 out: 5413 return notifier_from_errno(err); 5414 } 5415 5416 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5417 struct fib_info *fi) 5418 { 5419 const struct fib_nh *nh = fib_info_nh(fi, 0); 5420 5421 return nh->fib_nh_gw_family || 5422 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5423 } 5424 5425 static int 5426 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5427 struct mlxsw_sp_nexthop_group *nh_grp) 5428 { 5429 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5430 struct mlxsw_sp_nexthop_group_info *nhgi; 5431 struct mlxsw_sp_nexthop *nh; 5432 int err, i; 5433 5434 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5435 if (!nhgi) 5436 return -ENOMEM; 5437 nh_grp->nhgi = nhgi; 5438 nhgi->nh_grp = nh_grp; 5439 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5440 nhgi->count = nhs; 5441 for (i = 0; i < nhgi->count; i++) { 5442 struct fib_nh *fib_nh; 5443 5444 nh = &nhgi->nexthops[i]; 5445 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5446 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5447 if (err) 5448 goto err_nexthop4_init; 5449 } 5450 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5451 if (err) 5452 goto err_group_inc; 5453 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5454 if (err) 5455 goto err_group_refresh; 5456 5457 return 0; 5458 5459 err_group_refresh: 5460 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5461 err_group_inc: 5462 i = nhgi->count; 5463 err_nexthop4_init: 5464 for (i--; i >= 0; i--) { 5465 nh = &nhgi->nexthops[i]; 5466 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5467 } 5468 kfree(nhgi); 5469 return err; 5470 } 5471 5472 static void 5473 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5474 struct mlxsw_sp_nexthop_group *nh_grp) 5475 { 5476 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5477 int i; 5478 5479 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5480 for (i = nhgi->count - 1; i >= 0; i--) { 5481 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5482 5483 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5484 } 5485 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5486 WARN_ON_ONCE(nhgi->adj_index_valid); 5487 kfree(nhgi); 5488 } 5489 5490 static struct mlxsw_sp_nexthop_group * 5491 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5492 { 5493 struct mlxsw_sp_nexthop_group *nh_grp; 5494 int err; 5495 5496 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5497 if (!nh_grp) 5498 return ERR_PTR(-ENOMEM); 5499 INIT_LIST_HEAD(&nh_grp->vr_list); 5500 err = rhashtable_init(&nh_grp->vr_ht, 5501 &mlxsw_sp_nexthop_group_vr_ht_params); 5502 if (err) 5503 goto err_nexthop_group_vr_ht_init; 5504 INIT_LIST_HEAD(&nh_grp->fib_list); 5505 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5506 nh_grp->ipv4.fi = fi; 5507 fib_info_hold(fi); 5508 5509 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5510 if (err) 5511 goto err_nexthop_group_info_init; 5512 5513 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5514 if (err) 5515 goto err_nexthop_group_insert; 5516 5517 nh_grp->can_destroy = true; 5518 5519 return nh_grp; 5520 5521 err_nexthop_group_insert: 5522 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5523 err_nexthop_group_info_init: 5524 fib_info_put(fi); 5525 rhashtable_destroy(&nh_grp->vr_ht); 5526 err_nexthop_group_vr_ht_init: 5527 kfree(nh_grp); 5528 return ERR_PTR(err); 5529 } 5530 5531 static void 5532 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5533 struct mlxsw_sp_nexthop_group *nh_grp) 5534 { 5535 if (!nh_grp->can_destroy) 5536 return; 5537 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5538 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5539 fib_info_put(nh_grp->ipv4.fi); 5540 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5541 rhashtable_destroy(&nh_grp->vr_ht); 5542 kfree(nh_grp); 5543 } 5544 5545 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5546 struct mlxsw_sp_fib_entry *fib_entry, 5547 struct fib_info *fi) 5548 { 5549 struct mlxsw_sp_nexthop_group *nh_grp; 5550 5551 if (fi->nh) { 5552 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5553 fi->nh->id); 5554 if (WARN_ON_ONCE(!nh_grp)) 5555 return -EINVAL; 5556 goto out; 5557 } 5558 5559 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5560 if (!nh_grp) { 5561 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5562 if (IS_ERR(nh_grp)) 5563 return PTR_ERR(nh_grp); 5564 } 5565 out: 5566 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5567 fib_entry->nh_group = nh_grp; 5568 return 0; 5569 } 5570 5571 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5572 struct mlxsw_sp_fib_entry *fib_entry) 5573 { 5574 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5575 5576 list_del(&fib_entry->nexthop_group_node); 5577 if (!list_empty(&nh_grp->fib_list)) 5578 return; 5579 5580 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5581 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5582 return; 5583 } 5584 5585 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5586 } 5587 5588 static bool 5589 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5590 { 5591 struct mlxsw_sp_fib4_entry *fib4_entry; 5592 5593 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5594 common); 5595 return !fib4_entry->dscp; 5596 } 5597 5598 static bool 5599 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5600 { 5601 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5602 5603 switch (fib_entry->fib_node->fib->proto) { 5604 case MLXSW_SP_L3_PROTO_IPV4: 5605 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5606 return false; 5607 break; 5608 case MLXSW_SP_L3_PROTO_IPV6: 5609 break; 5610 } 5611 5612 switch (fib_entry->type) { 5613 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5614 return !!nh_group->nhgi->adj_index_valid; 5615 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5616 return !!mlxsw_sp_nhgi_rif(nh_group->nhgi); 5617 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5618 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5619 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5620 return true; 5621 default: 5622 return false; 5623 } 5624 } 5625 5626 static struct mlxsw_sp_nexthop * 5627 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5628 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5629 { 5630 int i; 5631 5632 for (i = 0; i < nh_grp->nhgi->count; i++) { 5633 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5634 struct net_device *dev = mlxsw_sp_nexthop_dev(nh); 5635 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5636 5637 if (dev && dev == rt->fib6_nh->fib_nh_dev && 5638 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5639 &rt->fib6_nh->fib_nh_gw6)) 5640 return nh; 5641 } 5642 5643 return NULL; 5644 } 5645 5646 static void 5647 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5648 struct fib_entry_notifier_info *fen_info) 5649 { 5650 u32 *p_dst = (u32 *) &fen_info->dst; 5651 struct fib_rt_info fri; 5652 5653 fri.fi = fen_info->fi; 5654 fri.tb_id = fen_info->tb_id; 5655 fri.dst = cpu_to_be32(*p_dst); 5656 fri.dst_len = fen_info->dst_len; 5657 fri.dscp = fen_info->dscp; 5658 fri.type = fen_info->type; 5659 fri.offload = false; 5660 fri.trap = false; 5661 fri.offload_failed = true; 5662 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5663 } 5664 5665 static void 5666 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5667 struct mlxsw_sp_fib_entry *fib_entry) 5668 { 5669 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5670 int dst_len = fib_entry->fib_node->key.prefix_len; 5671 struct mlxsw_sp_fib4_entry *fib4_entry; 5672 struct fib_rt_info fri; 5673 bool should_offload; 5674 5675 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5676 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5677 common); 5678 fri.fi = fib4_entry->fi; 5679 fri.tb_id = fib4_entry->tb_id; 5680 fri.dst = cpu_to_be32(*p_dst); 5681 fri.dst_len = dst_len; 5682 fri.dscp = fib4_entry->dscp; 5683 fri.type = fib4_entry->type; 5684 fri.offload = should_offload; 5685 fri.trap = !should_offload; 5686 fri.offload_failed = false; 5687 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5688 } 5689 5690 static void 5691 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5692 struct mlxsw_sp_fib_entry *fib_entry) 5693 { 5694 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5695 int dst_len = fib_entry->fib_node->key.prefix_len; 5696 struct mlxsw_sp_fib4_entry *fib4_entry; 5697 struct fib_rt_info fri; 5698 5699 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5700 common); 5701 fri.fi = fib4_entry->fi; 5702 fri.tb_id = fib4_entry->tb_id; 5703 fri.dst = cpu_to_be32(*p_dst); 5704 fri.dst_len = dst_len; 5705 fri.dscp = fib4_entry->dscp; 5706 fri.type = fib4_entry->type; 5707 fri.offload = false; 5708 fri.trap = false; 5709 fri.offload_failed = false; 5710 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5711 } 5712 5713 #if IS_ENABLED(CONFIG_IPV6) 5714 static void 5715 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5716 struct fib6_info **rt_arr, 5717 unsigned int nrt6) 5718 { 5719 int i; 5720 5721 /* In IPv6 a multipath route is represented using multiple routes, so 5722 * we need to set the flags on all of them. 5723 */ 5724 for (i = 0; i < nrt6; i++) 5725 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5726 false, false, true); 5727 } 5728 #else 5729 static void 5730 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5731 struct fib6_info **rt_arr, 5732 unsigned int nrt6) 5733 { 5734 } 5735 #endif 5736 5737 #if IS_ENABLED(CONFIG_IPV6) 5738 static void 5739 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5740 struct mlxsw_sp_fib_entry *fib_entry) 5741 { 5742 struct mlxsw_sp_fib6_entry *fib6_entry; 5743 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5744 bool should_offload; 5745 5746 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5747 5748 /* In IPv6 a multipath route is represented using multiple routes, so 5749 * we need to set the flags on all of them. 5750 */ 5751 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5752 common); 5753 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5754 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5755 should_offload, !should_offload, false); 5756 } 5757 #else 5758 static void 5759 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5760 struct mlxsw_sp_fib_entry *fib_entry) 5761 { 5762 } 5763 #endif 5764 5765 #if IS_ENABLED(CONFIG_IPV6) 5766 static void 5767 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5768 struct mlxsw_sp_fib_entry *fib_entry) 5769 { 5770 struct mlxsw_sp_fib6_entry *fib6_entry; 5771 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5772 5773 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5774 common); 5775 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5776 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5777 false, false, false); 5778 } 5779 #else 5780 static void 5781 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5782 struct mlxsw_sp_fib_entry *fib_entry) 5783 { 5784 } 5785 #endif 5786 5787 static void 5788 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5789 struct mlxsw_sp_fib_entry *fib_entry) 5790 { 5791 switch (fib_entry->fib_node->fib->proto) { 5792 case MLXSW_SP_L3_PROTO_IPV4: 5793 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5794 break; 5795 case MLXSW_SP_L3_PROTO_IPV6: 5796 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5797 break; 5798 } 5799 } 5800 5801 static void 5802 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5803 struct mlxsw_sp_fib_entry *fib_entry) 5804 { 5805 switch (fib_entry->fib_node->fib->proto) { 5806 case MLXSW_SP_L3_PROTO_IPV4: 5807 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5808 break; 5809 case MLXSW_SP_L3_PROTO_IPV6: 5810 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5811 break; 5812 } 5813 } 5814 5815 static void 5816 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5817 struct mlxsw_sp_fib_entry *fib_entry, 5818 enum mlxsw_reg_ralue_op op) 5819 { 5820 switch (op) { 5821 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 5822 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5823 break; 5824 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 5825 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5826 break; 5827 default: 5828 break; 5829 } 5830 } 5831 5832 static void 5833 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 5834 const struct mlxsw_sp_fib_entry *fib_entry, 5835 enum mlxsw_reg_ralue_op op) 5836 { 5837 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5838 enum mlxsw_reg_ralxx_protocol proto; 5839 u32 *p_dip; 5840 5841 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 5842 5843 switch (fib->proto) { 5844 case MLXSW_SP_L3_PROTO_IPV4: 5845 p_dip = (u32 *) fib_entry->fib_node->key.addr; 5846 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 5847 fib_entry->fib_node->key.prefix_len, 5848 *p_dip); 5849 break; 5850 case MLXSW_SP_L3_PROTO_IPV6: 5851 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 5852 fib_entry->fib_node->key.prefix_len, 5853 fib_entry->fib_node->key.addr); 5854 break; 5855 } 5856 } 5857 5858 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5859 struct mlxsw_sp_fib_entry *fib_entry, 5860 enum mlxsw_reg_ralue_op op) 5861 { 5862 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5863 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5864 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5865 enum mlxsw_reg_ralue_trap_action trap_action; 5866 u16 trap_id = 0; 5867 u32 adjacency_index = 0; 5868 u16 ecmp_size = 0; 5869 5870 /* In case the nexthop group adjacency index is valid, use it 5871 * with provided ECMP size. Otherwise, setup trap and pass 5872 * traffic to kernel. 5873 */ 5874 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5875 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5876 adjacency_index = nhgi->adj_index; 5877 ecmp_size = nhgi->ecmp_size; 5878 } else if (!nhgi->adj_index_valid && nhgi->count && 5879 mlxsw_sp_nhgi_rif(nhgi)) { 5880 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5881 adjacency_index = mlxsw_sp->router->adj_trap_index; 5882 ecmp_size = 1; 5883 } else { 5884 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5885 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5886 } 5887 5888 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5889 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 5890 adjacency_index, ecmp_size); 5891 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5892 } 5893 5894 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5895 struct mlxsw_sp_fib_entry *fib_entry, 5896 enum mlxsw_reg_ralue_op op) 5897 { 5898 struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi); 5899 enum mlxsw_reg_ralue_trap_action trap_action; 5900 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5901 u16 trap_id = 0; 5902 u16 rif_index = 0; 5903 5904 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5905 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5906 rif_index = rif->rif_index; 5907 } else { 5908 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5909 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5910 } 5911 5912 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5913 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 5914 rif_index); 5915 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5916 } 5917 5918 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5919 struct mlxsw_sp_fib_entry *fib_entry, 5920 enum mlxsw_reg_ralue_op op) 5921 { 5922 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5923 5924 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5925 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 5926 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5927 } 5928 5929 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5930 struct mlxsw_sp_fib_entry *fib_entry, 5931 enum mlxsw_reg_ralue_op op) 5932 { 5933 enum mlxsw_reg_ralue_trap_action trap_action; 5934 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5935 5936 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5937 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5938 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 5939 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5940 } 5941 5942 static int 5943 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5944 struct mlxsw_sp_fib_entry *fib_entry, 5945 enum mlxsw_reg_ralue_op op) 5946 { 5947 enum mlxsw_reg_ralue_trap_action trap_action; 5948 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5949 u16 trap_id; 5950 5951 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5952 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5953 5954 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5955 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0); 5956 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5957 } 5958 5959 static int 5960 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5961 struct mlxsw_sp_fib_entry *fib_entry, 5962 enum mlxsw_reg_ralue_op op) 5963 { 5964 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5965 const struct mlxsw_sp_ipip_ops *ipip_ops; 5966 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5967 int err; 5968 5969 if (WARN_ON(!ipip_entry)) 5970 return -EINVAL; 5971 5972 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5973 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5974 fib_entry->decap.tunnel_index); 5975 if (err) 5976 return err; 5977 5978 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5979 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 5980 fib_entry->decap.tunnel_index); 5981 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5982 } 5983 5984 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5985 struct mlxsw_sp_fib_entry *fib_entry, 5986 enum mlxsw_reg_ralue_op op) 5987 { 5988 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5989 5990 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5991 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 5992 fib_entry->decap.tunnel_index); 5993 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5994 } 5995 5996 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5997 struct mlxsw_sp_fib_entry *fib_entry, 5998 enum mlxsw_reg_ralue_op op) 5999 { 6000 switch (fib_entry->type) { 6001 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 6002 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 6003 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 6004 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 6005 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 6006 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 6007 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 6008 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 6009 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 6010 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry, 6011 op); 6012 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6013 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 6014 fib_entry, op); 6015 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 6016 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 6017 } 6018 return -EINVAL; 6019 } 6020 6021 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 6022 struct mlxsw_sp_fib_entry *fib_entry, 6023 enum mlxsw_reg_ralue_op op) 6024 { 6025 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 6026 6027 if (err) 6028 return err; 6029 6030 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 6031 6032 return err; 6033 } 6034 6035 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 6036 struct mlxsw_sp_fib_entry *fib_entry) 6037 { 6038 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6039 MLXSW_REG_RALUE_OP_WRITE_WRITE); 6040 } 6041 6042 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 6043 struct mlxsw_sp_fib_entry *fib_entry) 6044 { 6045 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 6046 MLXSW_REG_RALUE_OP_WRITE_DELETE); 6047 } 6048 6049 static int 6050 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6051 const struct fib_entry_notifier_info *fen_info, 6052 struct mlxsw_sp_fib_entry *fib_entry) 6053 { 6054 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6055 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 6056 struct mlxsw_sp_router *router = mlxsw_sp->router; 6057 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 6058 int ifindex = nhgi->nexthops[0].ifindex; 6059 struct mlxsw_sp_ipip_entry *ipip_entry; 6060 6061 switch (fen_info->type) { 6062 case RTN_LOCAL: 6063 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6064 MLXSW_SP_L3_PROTO_IPV4, dip); 6065 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6066 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6067 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 6068 fib_entry, 6069 ipip_entry); 6070 } 6071 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6072 MLXSW_SP_L3_PROTO_IPV4, 6073 &dip)) { 6074 u32 tunnel_index; 6075 6076 tunnel_index = router->nve_decap_config.tunnel_index; 6077 fib_entry->decap.tunnel_index = tunnel_index; 6078 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6079 return 0; 6080 } 6081 fallthrough; 6082 case RTN_BROADCAST: 6083 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6084 return 0; 6085 case RTN_BLACKHOLE: 6086 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6087 return 0; 6088 case RTN_UNREACHABLE: 6089 case RTN_PROHIBIT: 6090 /* Packets hitting these routes need to be trapped, but 6091 * can do so with a lower priority than packets directed 6092 * at the host, so use action type local instead of trap. 6093 */ 6094 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6095 return 0; 6096 case RTN_UNICAST: 6097 if (nhgi->gateway) 6098 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6099 else 6100 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6101 return 0; 6102 default: 6103 return -EINVAL; 6104 } 6105 } 6106 6107 static void 6108 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6109 struct mlxsw_sp_fib_entry *fib_entry) 6110 { 6111 switch (fib_entry->type) { 6112 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 6113 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 6114 break; 6115 default: 6116 break; 6117 } 6118 } 6119 6120 static void 6121 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6122 struct mlxsw_sp_fib4_entry *fib4_entry) 6123 { 6124 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 6125 } 6126 6127 static struct mlxsw_sp_fib4_entry * 6128 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 6129 struct mlxsw_sp_fib_node *fib_node, 6130 const struct fib_entry_notifier_info *fen_info) 6131 { 6132 struct mlxsw_sp_fib4_entry *fib4_entry; 6133 struct mlxsw_sp_fib_entry *fib_entry; 6134 int err; 6135 6136 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6137 if (!fib4_entry) 6138 return ERR_PTR(-ENOMEM); 6139 fib_entry = &fib4_entry->common; 6140 6141 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6142 if (err) 6143 goto err_nexthop4_group_get; 6144 6145 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6146 fib_node->fib); 6147 if (err) 6148 goto err_nexthop_group_vr_link; 6149 6150 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6151 if (err) 6152 goto err_fib4_entry_type_set; 6153 6154 fib4_entry->fi = fen_info->fi; 6155 fib_info_hold(fib4_entry->fi); 6156 fib4_entry->tb_id = fen_info->tb_id; 6157 fib4_entry->type = fen_info->type; 6158 fib4_entry->dscp = fen_info->dscp; 6159 6160 fib_entry->fib_node = fib_node; 6161 6162 return fib4_entry; 6163 6164 err_fib4_entry_type_set: 6165 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6166 err_nexthop_group_vr_link: 6167 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6168 err_nexthop4_group_get: 6169 kfree(fib4_entry); 6170 return ERR_PTR(err); 6171 } 6172 6173 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6174 struct mlxsw_sp_fib4_entry *fib4_entry) 6175 { 6176 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6177 6178 fib_info_put(fib4_entry->fi); 6179 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6180 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6181 fib_node->fib); 6182 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6183 kfree(fib4_entry); 6184 } 6185 6186 static struct mlxsw_sp_fib4_entry * 6187 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6188 const struct fib_entry_notifier_info *fen_info) 6189 { 6190 struct mlxsw_sp_fib4_entry *fib4_entry; 6191 struct mlxsw_sp_fib_node *fib_node; 6192 struct mlxsw_sp_fib *fib; 6193 struct mlxsw_sp_vr *vr; 6194 6195 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6196 if (!vr) 6197 return NULL; 6198 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6199 6200 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6201 sizeof(fen_info->dst), 6202 fen_info->dst_len); 6203 if (!fib_node) 6204 return NULL; 6205 6206 fib4_entry = container_of(fib_node->fib_entry, 6207 struct mlxsw_sp_fib4_entry, common); 6208 if (fib4_entry->tb_id == fen_info->tb_id && 6209 fib4_entry->dscp == fen_info->dscp && 6210 fib4_entry->type == fen_info->type && 6211 fib4_entry->fi == fen_info->fi) 6212 return fib4_entry; 6213 6214 return NULL; 6215 } 6216 6217 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6218 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6219 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6220 .key_len = sizeof(struct mlxsw_sp_fib_key), 6221 .automatic_shrinking = true, 6222 }; 6223 6224 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6225 struct mlxsw_sp_fib_node *fib_node) 6226 { 6227 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6228 mlxsw_sp_fib_ht_params); 6229 } 6230 6231 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6232 struct mlxsw_sp_fib_node *fib_node) 6233 { 6234 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6235 mlxsw_sp_fib_ht_params); 6236 } 6237 6238 static struct mlxsw_sp_fib_node * 6239 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6240 size_t addr_len, unsigned char prefix_len) 6241 { 6242 struct mlxsw_sp_fib_key key; 6243 6244 memset(&key, 0, sizeof(key)); 6245 memcpy(key.addr, addr, addr_len); 6246 key.prefix_len = prefix_len; 6247 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6248 } 6249 6250 static struct mlxsw_sp_fib_node * 6251 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6252 size_t addr_len, unsigned char prefix_len) 6253 { 6254 struct mlxsw_sp_fib_node *fib_node; 6255 6256 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6257 if (!fib_node) 6258 return NULL; 6259 6260 list_add(&fib_node->list, &fib->node_list); 6261 memcpy(fib_node->key.addr, addr, addr_len); 6262 fib_node->key.prefix_len = prefix_len; 6263 6264 return fib_node; 6265 } 6266 6267 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6268 { 6269 list_del(&fib_node->list); 6270 kfree(fib_node); 6271 } 6272 6273 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6274 struct mlxsw_sp_fib_node *fib_node) 6275 { 6276 struct mlxsw_sp_prefix_usage req_prefix_usage; 6277 struct mlxsw_sp_fib *fib = fib_node->fib; 6278 struct mlxsw_sp_lpm_tree *lpm_tree; 6279 int err; 6280 6281 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6282 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6283 goto out; 6284 6285 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6286 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6287 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6288 fib->proto); 6289 if (IS_ERR(lpm_tree)) 6290 return PTR_ERR(lpm_tree); 6291 6292 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6293 if (err) 6294 goto err_lpm_tree_replace; 6295 6296 out: 6297 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6298 return 0; 6299 6300 err_lpm_tree_replace: 6301 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6302 return err; 6303 } 6304 6305 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6306 struct mlxsw_sp_fib_node *fib_node) 6307 { 6308 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6309 struct mlxsw_sp_prefix_usage req_prefix_usage; 6310 struct mlxsw_sp_fib *fib = fib_node->fib; 6311 int err; 6312 6313 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6314 return; 6315 /* Try to construct a new LPM tree from the current prefix usage 6316 * minus the unused one. If we fail, continue using the old one. 6317 */ 6318 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6319 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6320 fib_node->key.prefix_len); 6321 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6322 fib->proto); 6323 if (IS_ERR(lpm_tree)) 6324 return; 6325 6326 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6327 if (err) 6328 goto err_lpm_tree_replace; 6329 6330 return; 6331 6332 err_lpm_tree_replace: 6333 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6334 } 6335 6336 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6337 struct mlxsw_sp_fib_node *fib_node, 6338 struct mlxsw_sp_fib *fib) 6339 { 6340 int err; 6341 6342 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6343 if (err) 6344 return err; 6345 fib_node->fib = fib; 6346 6347 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6348 if (err) 6349 goto err_fib_lpm_tree_link; 6350 6351 return 0; 6352 6353 err_fib_lpm_tree_link: 6354 fib_node->fib = NULL; 6355 mlxsw_sp_fib_node_remove(fib, fib_node); 6356 return err; 6357 } 6358 6359 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6360 struct mlxsw_sp_fib_node *fib_node) 6361 { 6362 struct mlxsw_sp_fib *fib = fib_node->fib; 6363 6364 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6365 fib_node->fib = NULL; 6366 mlxsw_sp_fib_node_remove(fib, fib_node); 6367 } 6368 6369 static struct mlxsw_sp_fib_node * 6370 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6371 size_t addr_len, unsigned char prefix_len, 6372 enum mlxsw_sp_l3proto proto) 6373 { 6374 struct mlxsw_sp_fib_node *fib_node; 6375 struct mlxsw_sp_fib *fib; 6376 struct mlxsw_sp_vr *vr; 6377 int err; 6378 6379 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6380 if (IS_ERR(vr)) 6381 return ERR_CAST(vr); 6382 fib = mlxsw_sp_vr_fib(vr, proto); 6383 6384 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6385 if (fib_node) 6386 return fib_node; 6387 6388 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6389 if (!fib_node) { 6390 err = -ENOMEM; 6391 goto err_fib_node_create; 6392 } 6393 6394 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6395 if (err) 6396 goto err_fib_node_init; 6397 6398 return fib_node; 6399 6400 err_fib_node_init: 6401 mlxsw_sp_fib_node_destroy(fib_node); 6402 err_fib_node_create: 6403 mlxsw_sp_vr_put(mlxsw_sp, vr); 6404 return ERR_PTR(err); 6405 } 6406 6407 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6408 struct mlxsw_sp_fib_node *fib_node) 6409 { 6410 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6411 6412 if (fib_node->fib_entry) 6413 return; 6414 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6415 mlxsw_sp_fib_node_destroy(fib_node); 6416 mlxsw_sp_vr_put(mlxsw_sp, vr); 6417 } 6418 6419 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6420 struct mlxsw_sp_fib_entry *fib_entry) 6421 { 6422 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6423 int err; 6424 6425 fib_node->fib_entry = fib_entry; 6426 6427 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 6428 if (err) 6429 goto err_fib_entry_update; 6430 6431 return 0; 6432 6433 err_fib_entry_update: 6434 fib_node->fib_entry = NULL; 6435 return err; 6436 } 6437 6438 static void 6439 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6440 struct mlxsw_sp_fib_entry *fib_entry) 6441 { 6442 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6443 6444 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 6445 fib_node->fib_entry = NULL; 6446 } 6447 6448 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6449 { 6450 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6451 struct mlxsw_sp_fib4_entry *fib4_replaced; 6452 6453 if (!fib_node->fib_entry) 6454 return true; 6455 6456 fib4_replaced = container_of(fib_node->fib_entry, 6457 struct mlxsw_sp_fib4_entry, common); 6458 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6459 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6460 return false; 6461 6462 return true; 6463 } 6464 6465 static int 6466 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6467 const struct fib_entry_notifier_info *fen_info) 6468 { 6469 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6470 struct mlxsw_sp_fib_entry *replaced; 6471 struct mlxsw_sp_fib_node *fib_node; 6472 int err; 6473 6474 if (fen_info->fi->nh && 6475 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6476 return 0; 6477 6478 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6479 &fen_info->dst, sizeof(fen_info->dst), 6480 fen_info->dst_len, 6481 MLXSW_SP_L3_PROTO_IPV4); 6482 if (IS_ERR(fib_node)) { 6483 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6484 return PTR_ERR(fib_node); 6485 } 6486 6487 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6488 if (IS_ERR(fib4_entry)) { 6489 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6490 err = PTR_ERR(fib4_entry); 6491 goto err_fib4_entry_create; 6492 } 6493 6494 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6495 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6496 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6497 return 0; 6498 } 6499 6500 replaced = fib_node->fib_entry; 6501 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common); 6502 if (err) { 6503 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6504 goto err_fib_node_entry_link; 6505 } 6506 6507 /* Nothing to replace */ 6508 if (!replaced) 6509 return 0; 6510 6511 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6512 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6513 common); 6514 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6515 6516 return 0; 6517 6518 err_fib_node_entry_link: 6519 fib_node->fib_entry = replaced; 6520 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6521 err_fib4_entry_create: 6522 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6523 return err; 6524 } 6525 6526 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6527 struct fib_entry_notifier_info *fen_info) 6528 { 6529 struct mlxsw_sp_fib4_entry *fib4_entry; 6530 struct mlxsw_sp_fib_node *fib_node; 6531 6532 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6533 if (!fib4_entry) 6534 return; 6535 fib_node = fib4_entry->common.fib_node; 6536 6537 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common); 6538 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6539 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6540 } 6541 6542 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6543 { 6544 /* Multicast routes aren't supported, so ignore them. Neighbour 6545 * Discovery packets are specifically trapped. 6546 */ 6547 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6548 return true; 6549 6550 /* Cloned routes are irrelevant in the forwarding path. */ 6551 if (rt->fib6_flags & RTF_CACHE) 6552 return true; 6553 6554 return false; 6555 } 6556 6557 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6558 { 6559 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6560 6561 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6562 if (!mlxsw_sp_rt6) 6563 return ERR_PTR(-ENOMEM); 6564 6565 /* In case of route replace, replaced route is deleted with 6566 * no notification. Take reference to prevent accessing freed 6567 * memory. 6568 */ 6569 mlxsw_sp_rt6->rt = rt; 6570 fib6_info_hold(rt); 6571 6572 return mlxsw_sp_rt6; 6573 } 6574 6575 #if IS_ENABLED(CONFIG_IPV6) 6576 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6577 { 6578 fib6_info_release(rt); 6579 } 6580 #else 6581 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6582 { 6583 } 6584 #endif 6585 6586 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6587 { 6588 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6589 6590 if (!mlxsw_sp_rt6->rt->nh) 6591 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6592 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6593 kfree(mlxsw_sp_rt6); 6594 } 6595 6596 static struct fib6_info * 6597 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6598 { 6599 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6600 list)->rt; 6601 } 6602 6603 static struct mlxsw_sp_rt6 * 6604 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6605 const struct fib6_info *rt) 6606 { 6607 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6608 6609 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6610 if (mlxsw_sp_rt6->rt == rt) 6611 return mlxsw_sp_rt6; 6612 } 6613 6614 return NULL; 6615 } 6616 6617 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6618 const struct fib6_info *rt, 6619 enum mlxsw_sp_ipip_type *ret) 6620 { 6621 return rt->fib6_nh->fib_nh_dev && 6622 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6623 } 6624 6625 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6626 struct mlxsw_sp_nexthop_group *nh_grp, 6627 struct mlxsw_sp_nexthop *nh, 6628 const struct fib6_info *rt) 6629 { 6630 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6631 int err; 6632 6633 nh->nhgi = nh_grp->nhgi; 6634 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6635 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6636 #if IS_ENABLED(CONFIG_IPV6) 6637 nh->neigh_tbl = &nd_tbl; 6638 #endif 6639 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6640 6641 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6642 6643 if (!dev) 6644 return 0; 6645 nh->ifindex = dev->ifindex; 6646 6647 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6648 if (err) 6649 goto err_nexthop_type_init; 6650 6651 return 0; 6652 6653 err_nexthop_type_init: 6654 list_del(&nh->router_list_node); 6655 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6656 return err; 6657 } 6658 6659 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6660 struct mlxsw_sp_nexthop *nh) 6661 { 6662 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6663 list_del(&nh->router_list_node); 6664 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6665 } 6666 6667 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6668 const struct fib6_info *rt) 6669 { 6670 return rt->fib6_nh->fib_nh_gw_family || 6671 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6672 } 6673 6674 static int 6675 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6676 struct mlxsw_sp_nexthop_group *nh_grp, 6677 struct mlxsw_sp_fib6_entry *fib6_entry) 6678 { 6679 struct mlxsw_sp_nexthop_group_info *nhgi; 6680 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6681 struct mlxsw_sp_nexthop *nh; 6682 int err, i; 6683 6684 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6685 GFP_KERNEL); 6686 if (!nhgi) 6687 return -ENOMEM; 6688 nh_grp->nhgi = nhgi; 6689 nhgi->nh_grp = nh_grp; 6690 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6691 struct mlxsw_sp_rt6, list); 6692 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6693 nhgi->count = fib6_entry->nrt6; 6694 for (i = 0; i < nhgi->count; i++) { 6695 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6696 6697 nh = &nhgi->nexthops[i]; 6698 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6699 if (err) 6700 goto err_nexthop6_init; 6701 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6702 } 6703 nh_grp->nhgi = nhgi; 6704 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6705 if (err) 6706 goto err_group_inc; 6707 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6708 if (err) 6709 goto err_group_refresh; 6710 6711 return 0; 6712 6713 err_group_refresh: 6714 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6715 err_group_inc: 6716 i = nhgi->count; 6717 err_nexthop6_init: 6718 for (i--; i >= 0; i--) { 6719 nh = &nhgi->nexthops[i]; 6720 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6721 } 6722 kfree(nhgi); 6723 return err; 6724 } 6725 6726 static void 6727 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6728 struct mlxsw_sp_nexthop_group *nh_grp) 6729 { 6730 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6731 int i; 6732 6733 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6734 for (i = nhgi->count - 1; i >= 0; i--) { 6735 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6736 6737 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6738 } 6739 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6740 WARN_ON_ONCE(nhgi->adj_index_valid); 6741 kfree(nhgi); 6742 } 6743 6744 static struct mlxsw_sp_nexthop_group * 6745 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6746 struct mlxsw_sp_fib6_entry *fib6_entry) 6747 { 6748 struct mlxsw_sp_nexthop_group *nh_grp; 6749 int err; 6750 6751 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6752 if (!nh_grp) 6753 return ERR_PTR(-ENOMEM); 6754 INIT_LIST_HEAD(&nh_grp->vr_list); 6755 err = rhashtable_init(&nh_grp->vr_ht, 6756 &mlxsw_sp_nexthop_group_vr_ht_params); 6757 if (err) 6758 goto err_nexthop_group_vr_ht_init; 6759 INIT_LIST_HEAD(&nh_grp->fib_list); 6760 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6761 6762 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6763 if (err) 6764 goto err_nexthop_group_info_init; 6765 6766 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6767 if (err) 6768 goto err_nexthop_group_insert; 6769 6770 nh_grp->can_destroy = true; 6771 6772 return nh_grp; 6773 6774 err_nexthop_group_insert: 6775 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6776 err_nexthop_group_info_init: 6777 rhashtable_destroy(&nh_grp->vr_ht); 6778 err_nexthop_group_vr_ht_init: 6779 kfree(nh_grp); 6780 return ERR_PTR(err); 6781 } 6782 6783 static void 6784 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6785 struct mlxsw_sp_nexthop_group *nh_grp) 6786 { 6787 if (!nh_grp->can_destroy) 6788 return; 6789 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6790 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6791 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6792 rhashtable_destroy(&nh_grp->vr_ht); 6793 kfree(nh_grp); 6794 } 6795 6796 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6797 struct mlxsw_sp_fib6_entry *fib6_entry) 6798 { 6799 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6800 struct mlxsw_sp_nexthop_group *nh_grp; 6801 6802 if (rt->nh) { 6803 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6804 rt->nh->id); 6805 if (WARN_ON_ONCE(!nh_grp)) 6806 return -EINVAL; 6807 goto out; 6808 } 6809 6810 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6811 if (!nh_grp) { 6812 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6813 if (IS_ERR(nh_grp)) 6814 return PTR_ERR(nh_grp); 6815 } 6816 6817 /* The route and the nexthop are described by the same struct, so we 6818 * need to the update the nexthop offload indication for the new route. 6819 */ 6820 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6821 6822 out: 6823 list_add_tail(&fib6_entry->common.nexthop_group_node, 6824 &nh_grp->fib_list); 6825 fib6_entry->common.nh_group = nh_grp; 6826 6827 return 0; 6828 } 6829 6830 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6831 struct mlxsw_sp_fib_entry *fib_entry) 6832 { 6833 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6834 6835 list_del(&fib_entry->nexthop_group_node); 6836 if (!list_empty(&nh_grp->fib_list)) 6837 return; 6838 6839 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6840 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6841 return; 6842 } 6843 6844 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6845 } 6846 6847 static int 6848 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6849 struct mlxsw_sp_fib6_entry *fib6_entry) 6850 { 6851 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6852 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6853 int err; 6854 6855 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6856 fib6_entry->common.nh_group = NULL; 6857 list_del(&fib6_entry->common.nexthop_group_node); 6858 6859 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6860 if (err) 6861 goto err_nexthop6_group_get; 6862 6863 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6864 fib_node->fib); 6865 if (err) 6866 goto err_nexthop_group_vr_link; 6867 6868 /* In case this entry is offloaded, then the adjacency index 6869 * currently associated with it in the device's table is that 6870 * of the old group. Start using the new one instead. 6871 */ 6872 err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common); 6873 if (err) 6874 goto err_fib_entry_update; 6875 6876 if (list_empty(&old_nh_grp->fib_list)) 6877 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6878 6879 return 0; 6880 6881 err_fib_entry_update: 6882 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6883 fib_node->fib); 6884 err_nexthop_group_vr_link: 6885 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6886 err_nexthop6_group_get: 6887 list_add_tail(&fib6_entry->common.nexthop_group_node, 6888 &old_nh_grp->fib_list); 6889 fib6_entry->common.nh_group = old_nh_grp; 6890 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6891 return err; 6892 } 6893 6894 static int 6895 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6896 struct mlxsw_sp_fib6_entry *fib6_entry, 6897 struct fib6_info **rt_arr, unsigned int nrt6) 6898 { 6899 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6900 int err, i; 6901 6902 for (i = 0; i < nrt6; i++) { 6903 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6904 if (IS_ERR(mlxsw_sp_rt6)) { 6905 err = PTR_ERR(mlxsw_sp_rt6); 6906 goto err_rt6_unwind; 6907 } 6908 6909 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6910 fib6_entry->nrt6++; 6911 } 6912 6913 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 6914 if (err) 6915 goto err_rt6_unwind; 6916 6917 return 0; 6918 6919 err_rt6_unwind: 6920 for (; i > 0; i--) { 6921 fib6_entry->nrt6--; 6922 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6923 struct mlxsw_sp_rt6, list); 6924 list_del(&mlxsw_sp_rt6->list); 6925 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6926 } 6927 return err; 6928 } 6929 6930 static void 6931 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6932 struct mlxsw_sp_fib6_entry *fib6_entry, 6933 struct fib6_info **rt_arr, unsigned int nrt6) 6934 { 6935 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6936 int i; 6937 6938 for (i = 0; i < nrt6; i++) { 6939 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6940 rt_arr[i]); 6941 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6942 continue; 6943 6944 fib6_entry->nrt6--; 6945 list_del(&mlxsw_sp_rt6->list); 6946 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6947 } 6948 6949 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 6950 } 6951 6952 static int 6953 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 6954 struct mlxsw_sp_fib_entry *fib_entry, 6955 const struct fib6_info *rt) 6956 { 6957 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6958 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 6959 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 6960 struct mlxsw_sp_router *router = mlxsw_sp->router; 6961 int ifindex = nhgi->nexthops[0].ifindex; 6962 struct mlxsw_sp_ipip_entry *ipip_entry; 6963 6964 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6965 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6966 MLXSW_SP_L3_PROTO_IPV6, 6967 dip); 6968 6969 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6970 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6971 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 6972 ipip_entry); 6973 } 6974 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6975 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 6976 u32 tunnel_index; 6977 6978 tunnel_index = router->nve_decap_config.tunnel_index; 6979 fib_entry->decap.tunnel_index = tunnel_index; 6980 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6981 } 6982 6983 return 0; 6984 } 6985 6986 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6987 struct mlxsw_sp_fib_entry *fib_entry, 6988 const struct fib6_info *rt) 6989 { 6990 if (rt->fib6_flags & RTF_LOCAL) 6991 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 6992 rt); 6993 if (rt->fib6_flags & RTF_ANYCAST) 6994 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6995 else if (rt->fib6_type == RTN_BLACKHOLE) 6996 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6997 else if (rt->fib6_flags & RTF_REJECT) 6998 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6999 else if (fib_entry->nh_group->nhgi->gateway) 7000 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 7001 else 7002 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 7003 7004 return 0; 7005 } 7006 7007 static void 7008 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 7009 { 7010 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 7011 7012 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 7013 list) { 7014 fib6_entry->nrt6--; 7015 list_del(&mlxsw_sp_rt6->list); 7016 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7017 } 7018 } 7019 7020 static struct mlxsw_sp_fib6_entry * 7021 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 7022 struct mlxsw_sp_fib_node *fib_node, 7023 struct fib6_info **rt_arr, unsigned int nrt6) 7024 { 7025 struct mlxsw_sp_fib6_entry *fib6_entry; 7026 struct mlxsw_sp_fib_entry *fib_entry; 7027 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 7028 int err, i; 7029 7030 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 7031 if (!fib6_entry) 7032 return ERR_PTR(-ENOMEM); 7033 fib_entry = &fib6_entry->common; 7034 7035 INIT_LIST_HEAD(&fib6_entry->rt6_list); 7036 7037 for (i = 0; i < nrt6; i++) { 7038 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 7039 if (IS_ERR(mlxsw_sp_rt6)) { 7040 err = PTR_ERR(mlxsw_sp_rt6); 7041 goto err_rt6_unwind; 7042 } 7043 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 7044 fib6_entry->nrt6++; 7045 } 7046 7047 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 7048 if (err) 7049 goto err_rt6_unwind; 7050 7051 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 7052 fib_node->fib); 7053 if (err) 7054 goto err_nexthop_group_vr_link; 7055 7056 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 7057 if (err) 7058 goto err_fib6_entry_type_set; 7059 7060 fib_entry->fib_node = fib_node; 7061 7062 return fib6_entry; 7063 7064 err_fib6_entry_type_set: 7065 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 7066 err_nexthop_group_vr_link: 7067 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 7068 err_rt6_unwind: 7069 for (; i > 0; i--) { 7070 fib6_entry->nrt6--; 7071 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 7072 struct mlxsw_sp_rt6, list); 7073 list_del(&mlxsw_sp_rt6->list); 7074 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 7075 } 7076 kfree(fib6_entry); 7077 return ERR_PTR(err); 7078 } 7079 7080 static void 7081 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 7082 struct mlxsw_sp_fib6_entry *fib6_entry) 7083 { 7084 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 7085 } 7086 7087 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 7088 struct mlxsw_sp_fib6_entry *fib6_entry) 7089 { 7090 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7091 7092 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 7093 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 7094 fib_node->fib); 7095 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 7096 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 7097 WARN_ON(fib6_entry->nrt6); 7098 kfree(fib6_entry); 7099 } 7100 7101 static struct mlxsw_sp_fib6_entry * 7102 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 7103 const struct fib6_info *rt) 7104 { 7105 struct mlxsw_sp_fib6_entry *fib6_entry; 7106 struct mlxsw_sp_fib_node *fib_node; 7107 struct mlxsw_sp_fib *fib; 7108 struct fib6_info *cmp_rt; 7109 struct mlxsw_sp_vr *vr; 7110 7111 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 7112 if (!vr) 7113 return NULL; 7114 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 7115 7116 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 7117 sizeof(rt->fib6_dst.addr), 7118 rt->fib6_dst.plen); 7119 if (!fib_node) 7120 return NULL; 7121 7122 fib6_entry = container_of(fib_node->fib_entry, 7123 struct mlxsw_sp_fib6_entry, common); 7124 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7125 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 7126 rt->fib6_metric == cmp_rt->fib6_metric && 7127 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 7128 return fib6_entry; 7129 7130 return NULL; 7131 } 7132 7133 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 7134 { 7135 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 7136 struct mlxsw_sp_fib6_entry *fib6_replaced; 7137 struct fib6_info *rt, *rt_replaced; 7138 7139 if (!fib_node->fib_entry) 7140 return true; 7141 7142 fib6_replaced = container_of(fib_node->fib_entry, 7143 struct mlxsw_sp_fib6_entry, 7144 common); 7145 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7146 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7147 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7148 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7149 return false; 7150 7151 return true; 7152 } 7153 7154 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7155 struct fib6_info **rt_arr, 7156 unsigned int nrt6) 7157 { 7158 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7159 struct mlxsw_sp_fib_entry *replaced; 7160 struct mlxsw_sp_fib_node *fib_node; 7161 struct fib6_info *rt = rt_arr[0]; 7162 int err; 7163 7164 if (rt->fib6_src.plen) 7165 return -EINVAL; 7166 7167 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7168 return 0; 7169 7170 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7171 return 0; 7172 7173 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7174 &rt->fib6_dst.addr, 7175 sizeof(rt->fib6_dst.addr), 7176 rt->fib6_dst.plen, 7177 MLXSW_SP_L3_PROTO_IPV6); 7178 if (IS_ERR(fib_node)) 7179 return PTR_ERR(fib_node); 7180 7181 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7182 nrt6); 7183 if (IS_ERR(fib6_entry)) { 7184 err = PTR_ERR(fib6_entry); 7185 goto err_fib6_entry_create; 7186 } 7187 7188 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7189 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7190 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7191 return 0; 7192 } 7193 7194 replaced = fib_node->fib_entry; 7195 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common); 7196 if (err) 7197 goto err_fib_node_entry_link; 7198 7199 /* Nothing to replace */ 7200 if (!replaced) 7201 return 0; 7202 7203 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7204 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7205 common); 7206 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7207 7208 return 0; 7209 7210 err_fib_node_entry_link: 7211 fib_node->fib_entry = replaced; 7212 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7213 err_fib6_entry_create: 7214 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7215 return err; 7216 } 7217 7218 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7219 struct fib6_info **rt_arr, 7220 unsigned int nrt6) 7221 { 7222 struct mlxsw_sp_fib6_entry *fib6_entry; 7223 struct mlxsw_sp_fib_node *fib_node; 7224 struct fib6_info *rt = rt_arr[0]; 7225 int err; 7226 7227 if (rt->fib6_src.plen) 7228 return -EINVAL; 7229 7230 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7231 return 0; 7232 7233 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7234 &rt->fib6_dst.addr, 7235 sizeof(rt->fib6_dst.addr), 7236 rt->fib6_dst.plen, 7237 MLXSW_SP_L3_PROTO_IPV6); 7238 if (IS_ERR(fib_node)) 7239 return PTR_ERR(fib_node); 7240 7241 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7242 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7243 return -EINVAL; 7244 } 7245 7246 fib6_entry = container_of(fib_node->fib_entry, 7247 struct mlxsw_sp_fib6_entry, common); 7248 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr, 7249 nrt6); 7250 if (err) 7251 goto err_fib6_entry_nexthop_add; 7252 7253 return 0; 7254 7255 err_fib6_entry_nexthop_add: 7256 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7257 return err; 7258 } 7259 7260 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7261 struct fib6_info **rt_arr, 7262 unsigned int nrt6) 7263 { 7264 struct mlxsw_sp_fib6_entry *fib6_entry; 7265 struct mlxsw_sp_fib_node *fib_node; 7266 struct fib6_info *rt = rt_arr[0]; 7267 7268 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7269 return; 7270 7271 /* Multipath routes are first added to the FIB trie and only then 7272 * notified. If we vetoed the addition, we will get a delete 7273 * notification for a route we do not have. Therefore, do not warn if 7274 * route was not found. 7275 */ 7276 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7277 if (!fib6_entry) 7278 return; 7279 7280 /* If not all the nexthops are deleted, then only reduce the nexthop 7281 * group. 7282 */ 7283 if (nrt6 != fib6_entry->nrt6) { 7284 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr, 7285 nrt6); 7286 return; 7287 } 7288 7289 fib_node = fib6_entry->common.fib_node; 7290 7291 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common); 7292 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7293 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7294 } 7295 7296 static struct mlxsw_sp_mr_table * 7297 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7298 { 7299 if (family == RTNL_FAMILY_IPMR) 7300 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7301 else 7302 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7303 } 7304 7305 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7306 struct mfc_entry_notifier_info *men_info, 7307 bool replace) 7308 { 7309 struct mlxsw_sp_mr_table *mrt; 7310 struct mlxsw_sp_vr *vr; 7311 7312 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7313 if (IS_ERR(vr)) 7314 return PTR_ERR(vr); 7315 7316 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7317 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7318 } 7319 7320 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7321 struct mfc_entry_notifier_info *men_info) 7322 { 7323 struct mlxsw_sp_mr_table *mrt; 7324 struct mlxsw_sp_vr *vr; 7325 7326 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7327 if (WARN_ON(!vr)) 7328 return; 7329 7330 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7331 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7332 mlxsw_sp_vr_put(mlxsw_sp, vr); 7333 } 7334 7335 static int 7336 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7337 struct vif_entry_notifier_info *ven_info) 7338 { 7339 struct mlxsw_sp_mr_table *mrt; 7340 struct mlxsw_sp_rif *rif; 7341 struct mlxsw_sp_vr *vr; 7342 7343 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7344 if (IS_ERR(vr)) 7345 return PTR_ERR(vr); 7346 7347 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7348 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7349 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7350 ven_info->vif_index, 7351 ven_info->vif_flags, rif); 7352 } 7353 7354 static void 7355 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7356 struct vif_entry_notifier_info *ven_info) 7357 { 7358 struct mlxsw_sp_mr_table *mrt; 7359 struct mlxsw_sp_vr *vr; 7360 7361 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7362 if (WARN_ON(!vr)) 7363 return; 7364 7365 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7366 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7367 mlxsw_sp_vr_put(mlxsw_sp, vr); 7368 } 7369 7370 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7371 struct mlxsw_sp_fib_node *fib_node) 7372 { 7373 struct mlxsw_sp_fib4_entry *fib4_entry; 7374 7375 fib4_entry = container_of(fib_node->fib_entry, 7376 struct mlxsw_sp_fib4_entry, common); 7377 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7378 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7379 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7380 } 7381 7382 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7383 struct mlxsw_sp_fib_node *fib_node) 7384 { 7385 struct mlxsw_sp_fib6_entry *fib6_entry; 7386 7387 fib6_entry = container_of(fib_node->fib_entry, 7388 struct mlxsw_sp_fib6_entry, common); 7389 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7390 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7391 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7392 } 7393 7394 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7395 struct mlxsw_sp_fib_node *fib_node) 7396 { 7397 switch (fib_node->fib->proto) { 7398 case MLXSW_SP_L3_PROTO_IPV4: 7399 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7400 break; 7401 case MLXSW_SP_L3_PROTO_IPV6: 7402 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7403 break; 7404 } 7405 } 7406 7407 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7408 struct mlxsw_sp_vr *vr, 7409 enum mlxsw_sp_l3proto proto) 7410 { 7411 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7412 struct mlxsw_sp_fib_node *fib_node, *tmp; 7413 7414 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7415 bool do_break = &tmp->list == &fib->node_list; 7416 7417 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7418 if (do_break) 7419 break; 7420 } 7421 } 7422 7423 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7424 { 7425 int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 7426 int i, j; 7427 7428 for (i = 0; i < max_vrs; i++) { 7429 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7430 7431 if (!mlxsw_sp_vr_is_used(vr)) 7432 continue; 7433 7434 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7435 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7436 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7437 7438 /* If virtual router was only used for IPv4, then it's no 7439 * longer used. 7440 */ 7441 if (!mlxsw_sp_vr_is_used(vr)) 7442 continue; 7443 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7444 } 7445 } 7446 7447 struct mlxsw_sp_fib6_event_work { 7448 struct fib6_info **rt_arr; 7449 unsigned int nrt6; 7450 }; 7451 7452 struct mlxsw_sp_fib_event_work { 7453 struct work_struct work; 7454 union { 7455 struct mlxsw_sp_fib6_event_work fib6_work; 7456 struct fib_entry_notifier_info fen_info; 7457 struct fib_rule_notifier_info fr_info; 7458 struct fib_nh_notifier_info fnh_info; 7459 struct mfc_entry_notifier_info men_info; 7460 struct vif_entry_notifier_info ven_info; 7461 }; 7462 struct mlxsw_sp *mlxsw_sp; 7463 unsigned long event; 7464 }; 7465 7466 static int 7467 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work, 7468 struct fib6_entry_notifier_info *fen6_info) 7469 { 7470 struct fib6_info *rt = fen6_info->rt; 7471 struct fib6_info **rt_arr; 7472 struct fib6_info *iter; 7473 unsigned int nrt6; 7474 int i = 0; 7475 7476 nrt6 = fen6_info->nsiblings + 1; 7477 7478 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7479 if (!rt_arr) 7480 return -ENOMEM; 7481 7482 fib6_work->rt_arr = rt_arr; 7483 fib6_work->nrt6 = nrt6; 7484 7485 rt_arr[0] = rt; 7486 fib6_info_hold(rt); 7487 7488 if (!fen6_info->nsiblings) 7489 return 0; 7490 7491 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7492 if (i == fen6_info->nsiblings) 7493 break; 7494 7495 rt_arr[i + 1] = iter; 7496 fib6_info_hold(iter); 7497 i++; 7498 } 7499 WARN_ON_ONCE(i != fen6_info->nsiblings); 7500 7501 return 0; 7502 } 7503 7504 static void 7505 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work) 7506 { 7507 int i; 7508 7509 for (i = 0; i < fib6_work->nrt6; i++) 7510 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]); 7511 kfree(fib6_work->rt_arr); 7512 } 7513 7514 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 7515 { 7516 struct mlxsw_sp_fib_event_work *fib_work = 7517 container_of(work, struct mlxsw_sp_fib_event_work, work); 7518 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7519 int err; 7520 7521 mutex_lock(&mlxsw_sp->router->lock); 7522 mlxsw_sp_span_respin(mlxsw_sp); 7523 7524 switch (fib_work->event) { 7525 case FIB_EVENT_ENTRY_REPLACE: 7526 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, 7527 &fib_work->fen_info); 7528 if (err) { 7529 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7530 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7531 &fib_work->fen_info); 7532 } 7533 fib_info_put(fib_work->fen_info.fi); 7534 break; 7535 case FIB_EVENT_ENTRY_DEL: 7536 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 7537 fib_info_put(fib_work->fen_info.fi); 7538 break; 7539 case FIB_EVENT_NH_ADD: 7540 case FIB_EVENT_NH_DEL: 7541 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 7542 fib_work->fnh_info.fib_nh); 7543 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 7544 break; 7545 } 7546 mutex_unlock(&mlxsw_sp->router->lock); 7547 kfree(fib_work); 7548 } 7549 7550 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 7551 { 7552 struct mlxsw_sp_fib_event_work *fib_work = 7553 container_of(work, struct mlxsw_sp_fib_event_work, work); 7554 struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work; 7555 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7556 int err; 7557 7558 mutex_lock(&mlxsw_sp->router->lock); 7559 mlxsw_sp_span_respin(mlxsw_sp); 7560 7561 switch (fib_work->event) { 7562 case FIB_EVENT_ENTRY_REPLACE: 7563 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, 7564 fib6_work->rt_arr, 7565 fib6_work->nrt6); 7566 if (err) { 7567 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7568 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7569 fib6_work->rt_arr, 7570 fib6_work->nrt6); 7571 } 7572 mlxsw_sp_router_fib6_work_fini(fib6_work); 7573 break; 7574 case FIB_EVENT_ENTRY_APPEND: 7575 err = mlxsw_sp_router_fib6_append(mlxsw_sp, 7576 fib6_work->rt_arr, 7577 fib6_work->nrt6); 7578 if (err) { 7579 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7580 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7581 fib6_work->rt_arr, 7582 fib6_work->nrt6); 7583 } 7584 mlxsw_sp_router_fib6_work_fini(fib6_work); 7585 break; 7586 case FIB_EVENT_ENTRY_DEL: 7587 mlxsw_sp_router_fib6_del(mlxsw_sp, 7588 fib6_work->rt_arr, 7589 fib6_work->nrt6); 7590 mlxsw_sp_router_fib6_work_fini(fib6_work); 7591 break; 7592 } 7593 mutex_unlock(&mlxsw_sp->router->lock); 7594 kfree(fib_work); 7595 } 7596 7597 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 7598 { 7599 struct mlxsw_sp_fib_event_work *fib_work = 7600 container_of(work, struct mlxsw_sp_fib_event_work, work); 7601 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7602 bool replace; 7603 int err; 7604 7605 rtnl_lock(); 7606 mutex_lock(&mlxsw_sp->router->lock); 7607 switch (fib_work->event) { 7608 case FIB_EVENT_ENTRY_REPLACE: 7609 case FIB_EVENT_ENTRY_ADD: 7610 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 7611 7612 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 7613 replace); 7614 if (err) 7615 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7616 mr_cache_put(fib_work->men_info.mfc); 7617 break; 7618 case FIB_EVENT_ENTRY_DEL: 7619 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 7620 mr_cache_put(fib_work->men_info.mfc); 7621 break; 7622 case FIB_EVENT_VIF_ADD: 7623 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7624 &fib_work->ven_info); 7625 if (err) 7626 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7627 dev_put(fib_work->ven_info.dev); 7628 break; 7629 case FIB_EVENT_VIF_DEL: 7630 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 7631 &fib_work->ven_info); 7632 dev_put(fib_work->ven_info.dev); 7633 break; 7634 } 7635 mutex_unlock(&mlxsw_sp->router->lock); 7636 rtnl_unlock(); 7637 kfree(fib_work); 7638 } 7639 7640 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 7641 struct fib_notifier_info *info) 7642 { 7643 struct fib_entry_notifier_info *fen_info; 7644 struct fib_nh_notifier_info *fnh_info; 7645 7646 switch (fib_work->event) { 7647 case FIB_EVENT_ENTRY_REPLACE: 7648 case FIB_EVENT_ENTRY_DEL: 7649 fen_info = container_of(info, struct fib_entry_notifier_info, 7650 info); 7651 fib_work->fen_info = *fen_info; 7652 /* Take reference on fib_info to prevent it from being 7653 * freed while work is queued. Release it afterwards. 7654 */ 7655 fib_info_hold(fib_work->fen_info.fi); 7656 break; 7657 case FIB_EVENT_NH_ADD: 7658 case FIB_EVENT_NH_DEL: 7659 fnh_info = container_of(info, struct fib_nh_notifier_info, 7660 info); 7661 fib_work->fnh_info = *fnh_info; 7662 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 7663 break; 7664 } 7665 } 7666 7667 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 7668 struct fib_notifier_info *info) 7669 { 7670 struct fib6_entry_notifier_info *fen6_info; 7671 int err; 7672 7673 switch (fib_work->event) { 7674 case FIB_EVENT_ENTRY_REPLACE: 7675 case FIB_EVENT_ENTRY_APPEND: 7676 case FIB_EVENT_ENTRY_DEL: 7677 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7678 info); 7679 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work, 7680 fen6_info); 7681 if (err) 7682 return err; 7683 break; 7684 } 7685 7686 return 0; 7687 } 7688 7689 static void 7690 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 7691 struct fib_notifier_info *info) 7692 { 7693 switch (fib_work->event) { 7694 case FIB_EVENT_ENTRY_REPLACE: 7695 case FIB_EVENT_ENTRY_ADD: 7696 case FIB_EVENT_ENTRY_DEL: 7697 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 7698 mr_cache_hold(fib_work->men_info.mfc); 7699 break; 7700 case FIB_EVENT_VIF_ADD: 7701 case FIB_EVENT_VIF_DEL: 7702 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 7703 dev_hold(fib_work->ven_info.dev); 7704 break; 7705 } 7706 } 7707 7708 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7709 struct fib_notifier_info *info, 7710 struct mlxsw_sp *mlxsw_sp) 7711 { 7712 struct netlink_ext_ack *extack = info->extack; 7713 struct fib_rule_notifier_info *fr_info; 7714 struct fib_rule *rule; 7715 int err = 0; 7716 7717 /* nothing to do at the moment */ 7718 if (event == FIB_EVENT_RULE_DEL) 7719 return 0; 7720 7721 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7722 rule = fr_info->rule; 7723 7724 /* Rule only affects locally generated traffic */ 7725 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7726 return 0; 7727 7728 switch (info->family) { 7729 case AF_INET: 7730 if (!fib4_rule_default(rule) && !rule->l3mdev) 7731 err = -EOPNOTSUPP; 7732 break; 7733 case AF_INET6: 7734 if (!fib6_rule_default(rule) && !rule->l3mdev) 7735 err = -EOPNOTSUPP; 7736 break; 7737 case RTNL_FAMILY_IPMR: 7738 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7739 err = -EOPNOTSUPP; 7740 break; 7741 case RTNL_FAMILY_IP6MR: 7742 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7743 err = -EOPNOTSUPP; 7744 break; 7745 } 7746 7747 if (err < 0) 7748 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7749 7750 return err; 7751 } 7752 7753 /* Called with rcu_read_lock() */ 7754 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7755 unsigned long event, void *ptr) 7756 { 7757 struct mlxsw_sp_fib_event_work *fib_work; 7758 struct fib_notifier_info *info = ptr; 7759 struct mlxsw_sp_router *router; 7760 int err; 7761 7762 if ((info->family != AF_INET && info->family != AF_INET6 && 7763 info->family != RTNL_FAMILY_IPMR && 7764 info->family != RTNL_FAMILY_IP6MR)) 7765 return NOTIFY_DONE; 7766 7767 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7768 7769 switch (event) { 7770 case FIB_EVENT_RULE_ADD: 7771 case FIB_EVENT_RULE_DEL: 7772 err = mlxsw_sp_router_fib_rule_event(event, info, 7773 router->mlxsw_sp); 7774 return notifier_from_errno(err); 7775 case FIB_EVENT_ENTRY_ADD: 7776 case FIB_EVENT_ENTRY_REPLACE: 7777 case FIB_EVENT_ENTRY_APPEND: 7778 if (info->family == AF_INET) { 7779 struct fib_entry_notifier_info *fen_info = ptr; 7780 7781 if (fen_info->fi->fib_nh_is_v6) { 7782 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7783 return notifier_from_errno(-EINVAL); 7784 } 7785 } 7786 break; 7787 } 7788 7789 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 7790 if (!fib_work) 7791 return NOTIFY_BAD; 7792 7793 fib_work->mlxsw_sp = router->mlxsw_sp; 7794 fib_work->event = event; 7795 7796 switch (info->family) { 7797 case AF_INET: 7798 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 7799 mlxsw_sp_router_fib4_event(fib_work, info); 7800 break; 7801 case AF_INET6: 7802 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 7803 err = mlxsw_sp_router_fib6_event(fib_work, info); 7804 if (err) 7805 goto err_fib_event; 7806 break; 7807 case RTNL_FAMILY_IP6MR: 7808 case RTNL_FAMILY_IPMR: 7809 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 7810 mlxsw_sp_router_fibmr_event(fib_work, info); 7811 break; 7812 } 7813 7814 mlxsw_core_schedule_work(&fib_work->work); 7815 7816 return NOTIFY_DONE; 7817 7818 err_fib_event: 7819 kfree(fib_work); 7820 return NOTIFY_BAD; 7821 } 7822 7823 static struct mlxsw_sp_rif * 7824 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7825 const struct net_device *dev) 7826 { 7827 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 7828 int i; 7829 7830 for (i = 0; i < max_rifs; i++) 7831 if (mlxsw_sp->router->rifs[i] && 7832 mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev)) 7833 return mlxsw_sp->router->rifs[i]; 7834 7835 return NULL; 7836 } 7837 7838 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7839 const struct net_device *dev) 7840 { 7841 struct mlxsw_sp_rif *rif; 7842 7843 mutex_lock(&mlxsw_sp->router->lock); 7844 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7845 mutex_unlock(&mlxsw_sp->router->lock); 7846 7847 return rif; 7848 } 7849 7850 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7851 { 7852 struct mlxsw_sp_rif *rif; 7853 u16 vid = 0; 7854 7855 mutex_lock(&mlxsw_sp->router->lock); 7856 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7857 if (!rif) 7858 goto out; 7859 7860 /* We only return the VID for VLAN RIFs. Otherwise we return an 7861 * invalid value (0). 7862 */ 7863 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN) 7864 goto out; 7865 7866 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7867 7868 out: 7869 mutex_unlock(&mlxsw_sp->router->lock); 7870 return vid; 7871 } 7872 7873 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7874 { 7875 char ritr_pl[MLXSW_REG_RITR_LEN]; 7876 int err; 7877 7878 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7879 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7880 if (err) 7881 return err; 7882 7883 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7884 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7885 } 7886 7887 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7888 struct mlxsw_sp_rif *rif) 7889 { 7890 /* Signal to nexthop cleanup that the RIF is going away. */ 7891 rif->crif->rif = NULL; 7892 7893 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7894 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7895 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7896 } 7897 7898 static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 7899 { 7900 struct inet6_dev *inet6_dev; 7901 struct in_device *idev; 7902 7903 idev = __in_dev_get_rcu(dev); 7904 if (idev && idev->ifa_list) 7905 return false; 7906 7907 inet6_dev = __in6_dev_get(dev); 7908 if (inet6_dev && !list_empty(&inet6_dev->addr_list)) 7909 return false; 7910 7911 return true; 7912 } 7913 7914 static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev) 7915 { 7916 bool addr_list_empty; 7917 7918 rcu_read_lock(); 7919 addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev); 7920 rcu_read_unlock(); 7921 7922 return addr_list_empty; 7923 } 7924 7925 static bool 7926 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7927 unsigned long event) 7928 { 7929 bool addr_list_empty; 7930 7931 switch (event) { 7932 case NETDEV_UP: 7933 return rif == NULL; 7934 case NETDEV_DOWN: 7935 addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev); 7936 7937 /* macvlans do not have a RIF, but rather piggy back on the 7938 * RIF of their lower device. 7939 */ 7940 if (netif_is_macvlan(dev) && addr_list_empty) 7941 return true; 7942 7943 if (rif && addr_list_empty && 7944 !netif_is_l3_slave(mlxsw_sp_rif_dev(rif))) 7945 return true; 7946 /* It is possible we already removed the RIF ourselves 7947 * if it was assigned to a netdev that is now a bridge 7948 * or LAG slave. 7949 */ 7950 return false; 7951 } 7952 7953 return false; 7954 } 7955 7956 static enum mlxsw_sp_rif_type 7957 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7958 const struct net_device *dev) 7959 { 7960 enum mlxsw_sp_fid_type type; 7961 7962 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7963 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7964 7965 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7966 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7967 type = MLXSW_SP_FID_TYPE_8021Q; 7968 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7969 type = MLXSW_SP_FID_TYPE_8021Q; 7970 else if (netif_is_bridge_master(dev)) 7971 type = MLXSW_SP_FID_TYPE_8021D; 7972 else 7973 type = MLXSW_SP_FID_TYPE_RFID; 7974 7975 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7976 } 7977 7978 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index, 7979 u8 rif_entries) 7980 { 7981 *p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table, 7982 rif_entries); 7983 if (*p_rif_index == 0) 7984 return -ENOBUFS; 7985 *p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET; 7986 7987 /* RIF indexes must be aligned to the allocation size. */ 7988 WARN_ON_ONCE(*p_rif_index % rif_entries); 7989 7990 return 0; 7991 } 7992 7993 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 7994 u8 rif_entries) 7995 { 7996 gen_pool_free(mlxsw_sp->router->rifs_table, 7997 MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries); 7998 } 7999 8000 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 8001 u16 vr_id, 8002 struct mlxsw_sp_crif *crif) 8003 { 8004 struct net_device *l3_dev = crif ? crif->key.dev : NULL; 8005 struct mlxsw_sp_rif *rif; 8006 8007 rif = kzalloc(rif_size, GFP_KERNEL); 8008 if (!rif) 8009 return NULL; 8010 8011 INIT_LIST_HEAD(&rif->neigh_list); 8012 if (l3_dev) { 8013 ether_addr_copy(rif->addr, l3_dev->dev_addr); 8014 rif->mtu = l3_dev->mtu; 8015 } 8016 rif->vr_id = vr_id; 8017 rif->rif_index = rif_index; 8018 if (crif) { 8019 rif->crif = crif; 8020 crif->rif = rif; 8021 } 8022 8023 return rif; 8024 } 8025 8026 static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif) 8027 { 8028 WARN_ON(!list_empty(&rif->neigh_list)); 8029 8030 if (rif->crif) 8031 rif->crif->rif = NULL; 8032 kfree(rif); 8033 } 8034 8035 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 8036 u16 rif_index) 8037 { 8038 return mlxsw_sp->router->rifs[rif_index]; 8039 } 8040 8041 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 8042 { 8043 return rif->rif_index; 8044 } 8045 8046 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8047 { 8048 return lb_rif->common.rif_index; 8049 } 8050 8051 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8052 { 8053 struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common); 8054 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 8055 struct mlxsw_sp_vr *ul_vr; 8056 8057 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 8058 if (WARN_ON(IS_ERR(ul_vr))) 8059 return 0; 8060 8061 return ul_vr->id; 8062 } 8063 8064 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 8065 { 8066 return lb_rif->ul_rif_id; 8067 } 8068 8069 static bool 8070 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 8071 { 8072 return mlxsw_sp_rif_counter_valid_get(rif, 8073 MLXSW_SP_RIF_COUNTER_EGRESS) && 8074 mlxsw_sp_rif_counter_valid_get(rif, 8075 MLXSW_SP_RIF_COUNTER_INGRESS); 8076 } 8077 8078 static int 8079 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 8080 { 8081 int err; 8082 8083 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8084 if (err) 8085 return err; 8086 8087 /* Clear stale data. */ 8088 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8089 MLXSW_SP_RIF_COUNTER_INGRESS, 8090 NULL); 8091 if (err) 8092 goto err_clear_ingress; 8093 8094 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8095 if (err) 8096 goto err_alloc_egress; 8097 8098 /* Clear stale data. */ 8099 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8100 MLXSW_SP_RIF_COUNTER_EGRESS, 8101 NULL); 8102 if (err) 8103 goto err_clear_egress; 8104 8105 return 0; 8106 8107 err_clear_egress: 8108 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8109 err_alloc_egress: 8110 err_clear_ingress: 8111 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8112 return err; 8113 } 8114 8115 static void 8116 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 8117 { 8118 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 8119 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 8120 } 8121 8122 static void 8123 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 8124 struct netdev_notifier_offload_xstats_info *info) 8125 { 8126 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8127 return; 8128 netdev_offload_xstats_report_used(info->report_used); 8129 } 8130 8131 static int 8132 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 8133 struct rtnl_hw_stats64 *p_stats) 8134 { 8135 struct mlxsw_sp_rif_counter_set_basic ingress; 8136 struct mlxsw_sp_rif_counter_set_basic egress; 8137 int err; 8138 8139 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8140 MLXSW_SP_RIF_COUNTER_INGRESS, 8141 &ingress); 8142 if (err) 8143 return err; 8144 8145 err = mlxsw_sp_rif_counter_fetch_clear(rif, 8146 MLXSW_SP_RIF_COUNTER_EGRESS, 8147 &egress); 8148 if (err) 8149 return err; 8150 8151 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 8152 ((SET.good_unicast_ ## SFX) + \ 8153 (SET.good_multicast_ ## SFX) + \ 8154 (SET.good_broadcast_ ## SFX)) 8155 8156 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 8157 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 8158 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 8159 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 8160 p_stats->rx_errors = ingress.error_packets; 8161 p_stats->tx_errors = egress.error_packets; 8162 p_stats->rx_dropped = ingress.discard_packets; 8163 p_stats->tx_dropped = egress.discard_packets; 8164 p_stats->multicast = ingress.good_multicast_packets + 8165 ingress.good_broadcast_packets; 8166 8167 #undef MLXSW_SP_ROUTER_ALL_GOOD 8168 8169 return 0; 8170 } 8171 8172 static int 8173 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 8174 struct netdev_notifier_offload_xstats_info *info) 8175 { 8176 struct rtnl_hw_stats64 stats = {}; 8177 int err; 8178 8179 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 8180 return 0; 8181 8182 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8183 if (err) 8184 return err; 8185 8186 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8187 return 0; 8188 } 8189 8190 struct mlxsw_sp_router_hwstats_notify_work { 8191 struct work_struct work; 8192 struct net_device *dev; 8193 }; 8194 8195 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8196 { 8197 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8198 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8199 work); 8200 8201 rtnl_lock(); 8202 rtnl_offload_xstats_notify(hws_work->dev); 8203 rtnl_unlock(); 8204 dev_put(hws_work->dev); 8205 kfree(hws_work); 8206 } 8207 8208 static void 8209 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8210 { 8211 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8212 8213 /* To collect notification payload, the core ends up sending another 8214 * notifier block message, which would deadlock on the attempt to 8215 * acquire the router lock again. Just postpone the notification until 8216 * later. 8217 */ 8218 8219 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8220 if (!hws_work) 8221 return; 8222 8223 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8224 dev_hold(dev); 8225 hws_work->dev = dev; 8226 mlxsw_core_schedule_work(&hws_work->work); 8227 } 8228 8229 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8230 { 8231 return mlxsw_sp_rif_dev(rif)->ifindex; 8232 } 8233 8234 bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif) 8235 { 8236 return !!mlxsw_sp_rif_dev(rif); 8237 } 8238 8239 bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif, 8240 const struct net_device *dev) 8241 { 8242 return mlxsw_sp_rif_dev(rif) == dev; 8243 } 8244 8245 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8246 { 8247 struct rtnl_hw_stats64 stats = {}; 8248 8249 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8250 netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif), 8251 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8252 &stats); 8253 } 8254 8255 static struct mlxsw_sp_rif * 8256 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8257 const struct mlxsw_sp_rif_params *params, 8258 struct netlink_ext_ack *extack) 8259 { 8260 u8 rif_entries = params->double_entry ? 2 : 1; 8261 u32 tb_id = l3mdev_fib_table(params->dev); 8262 const struct mlxsw_sp_rif_ops *ops; 8263 struct mlxsw_sp_fid *fid = NULL; 8264 enum mlxsw_sp_rif_type type; 8265 struct mlxsw_sp_crif *crif; 8266 struct mlxsw_sp_rif *rif; 8267 struct mlxsw_sp_vr *vr; 8268 u16 rif_index; 8269 int i, err; 8270 8271 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8272 ops = mlxsw_sp->router->rif_ops_arr[type]; 8273 8274 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8275 if (IS_ERR(vr)) 8276 return ERR_CAST(vr); 8277 vr->rif_count++; 8278 8279 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 8280 if (err) { 8281 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8282 goto err_rif_index_alloc; 8283 } 8284 8285 crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev); 8286 if (WARN_ON(!crif)) { 8287 err = -ENOENT; 8288 goto err_crif_lookup; 8289 } 8290 8291 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif); 8292 if (!rif) { 8293 err = -ENOMEM; 8294 goto err_rif_alloc; 8295 } 8296 dev_hold(params->dev); 8297 mlxsw_sp->router->rifs[rif_index] = rif; 8298 rif->mlxsw_sp = mlxsw_sp; 8299 rif->ops = ops; 8300 rif->rif_entries = rif_entries; 8301 8302 if (ops->fid_get) { 8303 fid = ops->fid_get(rif, extack); 8304 if (IS_ERR(fid)) { 8305 err = PTR_ERR(fid); 8306 goto err_fid_get; 8307 } 8308 rif->fid = fid; 8309 } 8310 8311 if (ops->setup) 8312 ops->setup(rif, params); 8313 8314 err = ops->configure(rif, extack); 8315 if (err) 8316 goto err_configure; 8317 8318 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8319 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8320 if (err) 8321 goto err_mr_rif_add; 8322 } 8323 8324 if (netdev_offload_xstats_enabled(params->dev, 8325 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8326 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8327 if (err) 8328 goto err_stats_enable; 8329 mlxsw_sp_router_hwstats_notify_schedule(params->dev); 8330 } else { 8331 mlxsw_sp_rif_counters_alloc(rif); 8332 } 8333 8334 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 8335 return rif; 8336 8337 err_stats_enable: 8338 err_mr_rif_add: 8339 for (i--; i >= 0; i--) 8340 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8341 ops->deconfigure(rif); 8342 err_configure: 8343 if (fid) 8344 mlxsw_sp_fid_put(fid); 8345 err_fid_get: 8346 mlxsw_sp->router->rifs[rif_index] = NULL; 8347 dev_put(params->dev); 8348 mlxsw_sp_rif_free(rif); 8349 err_rif_alloc: 8350 err_crif_lookup: 8351 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8352 err_rif_index_alloc: 8353 vr->rif_count--; 8354 mlxsw_sp_vr_put(mlxsw_sp, vr); 8355 return ERR_PTR(err); 8356 } 8357 8358 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8359 { 8360 struct net_device *dev = mlxsw_sp_rif_dev(rif); 8361 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8362 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8363 struct mlxsw_sp_crif *crif = rif->crif; 8364 struct mlxsw_sp_fid *fid = rif->fid; 8365 u8 rif_entries = rif->rif_entries; 8366 u16 rif_index = rif->rif_index; 8367 struct mlxsw_sp_vr *vr; 8368 int i; 8369 8370 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 8371 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8372 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8373 8374 if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8375 mlxsw_sp_rif_push_l3_stats(rif); 8376 mlxsw_sp_router_port_l3_stats_disable(rif); 8377 mlxsw_sp_router_hwstats_notify_schedule(dev); 8378 } else { 8379 mlxsw_sp_rif_counters_free(rif); 8380 } 8381 8382 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8383 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8384 ops->deconfigure(rif); 8385 if (fid) 8386 /* Loopback RIFs are not associated with a FID. */ 8387 mlxsw_sp_fid_put(fid); 8388 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8389 dev_put(dev); 8390 mlxsw_sp_rif_free(rif); 8391 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 8392 vr->rif_count--; 8393 mlxsw_sp_vr_put(mlxsw_sp, vr); 8394 8395 if (crif->can_destroy) 8396 mlxsw_sp_crif_free(crif); 8397 } 8398 8399 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8400 struct net_device *dev) 8401 { 8402 struct mlxsw_sp_rif *rif; 8403 8404 mutex_lock(&mlxsw_sp->router->lock); 8405 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8406 if (!rif) 8407 goto out; 8408 mlxsw_sp_rif_destroy(rif); 8409 out: 8410 mutex_unlock(&mlxsw_sp->router->lock); 8411 } 8412 8413 static void 8414 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8415 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8416 { 8417 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8418 8419 params->vid = mlxsw_sp_port_vlan->vid; 8420 params->lag = mlxsw_sp_port->lagged; 8421 if (params->lag) 8422 params->lag_id = mlxsw_sp_port->lag_id; 8423 else 8424 params->system_port = mlxsw_sp_port->local_port; 8425 } 8426 8427 static struct mlxsw_sp_rif_subport * 8428 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8429 { 8430 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8431 } 8432 8433 static struct mlxsw_sp_rif * 8434 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8435 const struct mlxsw_sp_rif_params *params, 8436 struct netlink_ext_ack *extack) 8437 { 8438 struct mlxsw_sp_rif_subport *rif_subport; 8439 struct mlxsw_sp_rif *rif; 8440 8441 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8442 if (!rif) 8443 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8444 8445 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8446 refcount_inc(&rif_subport->ref_count); 8447 return rif; 8448 } 8449 8450 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8451 { 8452 struct mlxsw_sp_rif_subport *rif_subport; 8453 8454 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8455 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8456 return; 8457 8458 mlxsw_sp_rif_destroy(rif); 8459 } 8460 8461 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8462 struct mlxsw_sp_rif_mac_profile *profile, 8463 struct netlink_ext_ack *extack) 8464 { 8465 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8466 struct mlxsw_sp_router *router = mlxsw_sp->router; 8467 int id; 8468 8469 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8470 max_rif_mac_profiles, GFP_KERNEL); 8471 8472 if (id >= 0) { 8473 profile->id = id; 8474 return 0; 8475 } 8476 8477 if (id == -ENOSPC) 8478 NL_SET_ERR_MSG_MOD(extack, 8479 "Exceeded number of supported router interface MAC profiles"); 8480 8481 return id; 8482 } 8483 8484 static struct mlxsw_sp_rif_mac_profile * 8485 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8486 { 8487 struct mlxsw_sp_rif_mac_profile *profile; 8488 8489 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8490 mac_profile); 8491 WARN_ON(!profile); 8492 return profile; 8493 } 8494 8495 static struct mlxsw_sp_rif_mac_profile * 8496 mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8497 { 8498 struct mlxsw_sp_rif_mac_profile *profile; 8499 8500 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8501 if (!profile) 8502 return NULL; 8503 8504 ether_addr_copy(profile->mac_prefix, mac); 8505 refcount_set(&profile->ref_count, 1); 8506 return profile; 8507 } 8508 8509 static struct mlxsw_sp_rif_mac_profile * 8510 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8511 { 8512 struct mlxsw_sp_router *router = mlxsw_sp->router; 8513 struct mlxsw_sp_rif_mac_profile *profile; 8514 int id; 8515 8516 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8517 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8518 mlxsw_sp->mac_mask)) 8519 return profile; 8520 } 8521 8522 return NULL; 8523 } 8524 8525 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8526 { 8527 const struct mlxsw_sp *mlxsw_sp = priv; 8528 8529 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8530 } 8531 8532 static u64 mlxsw_sp_rifs_occ_get(void *priv) 8533 { 8534 const struct mlxsw_sp *mlxsw_sp = priv; 8535 8536 return atomic_read(&mlxsw_sp->router->rifs_count); 8537 } 8538 8539 static struct mlxsw_sp_rif_mac_profile * 8540 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8541 struct netlink_ext_ack *extack) 8542 { 8543 struct mlxsw_sp_rif_mac_profile *profile; 8544 int err; 8545 8546 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8547 if (!profile) 8548 return ERR_PTR(-ENOMEM); 8549 8550 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8551 if (err) 8552 goto profile_index_alloc_err; 8553 8554 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8555 return profile; 8556 8557 profile_index_alloc_err: 8558 kfree(profile); 8559 return ERR_PTR(err); 8560 } 8561 8562 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8563 u8 mac_profile) 8564 { 8565 struct mlxsw_sp_rif_mac_profile *profile; 8566 8567 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8568 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8569 kfree(profile); 8570 } 8571 8572 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8573 const char *mac, u8 *p_mac_profile, 8574 struct netlink_ext_ack *extack) 8575 { 8576 struct mlxsw_sp_rif_mac_profile *profile; 8577 8578 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8579 if (profile) { 8580 refcount_inc(&profile->ref_count); 8581 goto out; 8582 } 8583 8584 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8585 if (IS_ERR(profile)) 8586 return PTR_ERR(profile); 8587 8588 out: 8589 *p_mac_profile = profile->id; 8590 return 0; 8591 } 8592 8593 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8594 u8 mac_profile) 8595 { 8596 struct mlxsw_sp_rif_mac_profile *profile; 8597 8598 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8599 mac_profile); 8600 if (WARN_ON(!profile)) 8601 return; 8602 8603 if (!refcount_dec_and_test(&profile->ref_count)) 8604 return; 8605 8606 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8607 } 8608 8609 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8610 { 8611 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8612 struct mlxsw_sp_rif_mac_profile *profile; 8613 8614 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8615 rif->mac_profile_id); 8616 if (WARN_ON(!profile)) 8617 return false; 8618 8619 return refcount_read(&profile->ref_count) > 1; 8620 } 8621 8622 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8623 const char *new_mac) 8624 { 8625 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8626 struct mlxsw_sp_rif_mac_profile *profile; 8627 8628 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8629 rif->mac_profile_id); 8630 if (WARN_ON(!profile)) 8631 return -EINVAL; 8632 8633 ether_addr_copy(profile->mac_prefix, new_mac); 8634 return 0; 8635 } 8636 8637 static int 8638 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8639 struct mlxsw_sp_rif *rif, 8640 const char *new_mac, 8641 struct netlink_ext_ack *extack) 8642 { 8643 u8 mac_profile; 8644 int err; 8645 8646 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8647 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8648 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8649 8650 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8651 &mac_profile, extack); 8652 if (err) 8653 return err; 8654 8655 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8656 rif->mac_profile_id = mac_profile; 8657 return 0; 8658 } 8659 8660 static int 8661 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8662 struct net_device *l3_dev, 8663 struct netlink_ext_ack *extack) 8664 { 8665 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8666 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8667 struct mlxsw_sp_rif_params params = { 8668 .dev = l3_dev, 8669 }; 8670 u16 vid = mlxsw_sp_port_vlan->vid; 8671 struct mlxsw_sp_rif *rif; 8672 struct mlxsw_sp_fid *fid; 8673 int err; 8674 8675 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8676 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8677 if (IS_ERR(rif)) 8678 return PTR_ERR(rif); 8679 8680 /* FID was already created, just take a reference */ 8681 fid = rif->ops->fid_get(rif, extack); 8682 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8683 if (err) 8684 goto err_fid_port_vid_map; 8685 8686 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8687 if (err) 8688 goto err_port_vid_learning_set; 8689 8690 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8691 BR_STATE_FORWARDING); 8692 if (err) 8693 goto err_port_vid_stp_set; 8694 8695 mlxsw_sp_port_vlan->fid = fid; 8696 8697 return 0; 8698 8699 err_port_vid_stp_set: 8700 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8701 err_port_vid_learning_set: 8702 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8703 err_fid_port_vid_map: 8704 mlxsw_sp_fid_put(fid); 8705 mlxsw_sp_rif_subport_put(rif); 8706 return err; 8707 } 8708 8709 static void 8710 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8711 { 8712 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8713 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8714 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8715 u16 vid = mlxsw_sp_port_vlan->vid; 8716 8717 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8718 return; 8719 8720 mlxsw_sp_port_vlan->fid = NULL; 8721 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8722 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8723 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8724 mlxsw_sp_fid_put(fid); 8725 mlxsw_sp_rif_subport_put(rif); 8726 } 8727 8728 static int 8729 mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8730 struct net_device *l3_dev, 8731 struct netlink_ext_ack *extack) 8732 { 8733 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8734 8735 lockdep_assert_held(&mlxsw_sp->router->lock); 8736 8737 if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev)) 8738 return 0; 8739 8740 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8741 extack); 8742 } 8743 8744 void 8745 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8746 { 8747 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8748 8749 mutex_lock(&mlxsw_sp->router->lock); 8750 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8751 mutex_unlock(&mlxsw_sp->router->lock); 8752 } 8753 8754 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8755 struct net_device *port_dev, 8756 unsigned long event, u16 vid, 8757 struct netlink_ext_ack *extack) 8758 { 8759 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8760 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8761 8762 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8763 if (WARN_ON(!mlxsw_sp_port_vlan)) 8764 return -EINVAL; 8765 8766 switch (event) { 8767 case NETDEV_UP: 8768 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8769 l3_dev, extack); 8770 case NETDEV_DOWN: 8771 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8772 break; 8773 } 8774 8775 return 0; 8776 } 8777 8778 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8779 unsigned long event, 8780 struct netlink_ext_ack *extack) 8781 { 8782 if (netif_is_any_bridge_port(port_dev) || netif_is_lag_port(port_dev)) 8783 return 0; 8784 8785 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8786 MLXSW_SP_DEFAULT_VID, extack); 8787 } 8788 8789 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8790 struct net_device *lag_dev, 8791 unsigned long event, u16 vid, 8792 struct netlink_ext_ack *extack) 8793 { 8794 struct net_device *port_dev; 8795 struct list_head *iter; 8796 int err; 8797 8798 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8799 if (mlxsw_sp_port_dev_check(port_dev)) { 8800 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8801 port_dev, 8802 event, vid, 8803 extack); 8804 if (err) 8805 return err; 8806 } 8807 } 8808 8809 return 0; 8810 } 8811 8812 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8813 unsigned long event, 8814 struct netlink_ext_ack *extack) 8815 { 8816 if (netif_is_bridge_port(lag_dev)) 8817 return 0; 8818 8819 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8820 MLXSW_SP_DEFAULT_VID, extack); 8821 } 8822 8823 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8824 struct net_device *l3_dev, 8825 unsigned long event, 8826 struct netlink_ext_ack *extack) 8827 { 8828 struct mlxsw_sp_rif_params params = { 8829 .dev = l3_dev, 8830 }; 8831 struct mlxsw_sp_rif *rif; 8832 8833 switch (event) { 8834 case NETDEV_UP: 8835 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8836 u16 proto; 8837 8838 br_vlan_get_proto(l3_dev, &proto); 8839 if (proto == ETH_P_8021AD) { 8840 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8841 return -EOPNOTSUPP; 8842 } 8843 } 8844 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8845 if (IS_ERR(rif)) 8846 return PTR_ERR(rif); 8847 break; 8848 case NETDEV_DOWN: 8849 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8850 mlxsw_sp_rif_destroy(rif); 8851 break; 8852 } 8853 8854 return 0; 8855 } 8856 8857 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8858 struct net_device *vlan_dev, 8859 unsigned long event, 8860 struct netlink_ext_ack *extack) 8861 { 8862 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8863 u16 vid = vlan_dev_vlan_id(vlan_dev); 8864 8865 if (netif_is_bridge_port(vlan_dev)) 8866 return 0; 8867 8868 if (mlxsw_sp_port_dev_check(real_dev)) 8869 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8870 event, vid, extack); 8871 else if (netif_is_lag_master(real_dev)) 8872 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8873 vid, extack); 8874 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8875 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8876 extack); 8877 8878 return 0; 8879 } 8880 8881 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8882 { 8883 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8884 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8885 8886 return ether_addr_equal_masked(mac, vrrp4, mask); 8887 } 8888 8889 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8890 { 8891 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8892 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8893 8894 return ether_addr_equal_masked(mac, vrrp6, mask); 8895 } 8896 8897 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8898 const u8 *mac, bool adding) 8899 { 8900 char ritr_pl[MLXSW_REG_RITR_LEN]; 8901 u8 vrrp_id = adding ? mac[5] : 0; 8902 int err; 8903 8904 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8905 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8906 return 0; 8907 8908 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8909 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8910 if (err) 8911 return err; 8912 8913 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8914 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8915 else 8916 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8917 8918 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8919 } 8920 8921 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8922 const struct net_device *macvlan_dev, 8923 struct netlink_ext_ack *extack) 8924 { 8925 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8926 struct mlxsw_sp_rif *rif; 8927 int err; 8928 8929 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8930 if (!rif) { 8931 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8932 return -EOPNOTSUPP; 8933 } 8934 8935 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8936 mlxsw_sp_fid_index(rif->fid), true); 8937 if (err) 8938 return err; 8939 8940 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8941 macvlan_dev->dev_addr, true); 8942 if (err) 8943 goto err_rif_vrrp_add; 8944 8945 /* Make sure the bridge driver does not have this MAC pointing at 8946 * some other port. 8947 */ 8948 if (rif->ops->fdb_del) 8949 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8950 8951 return 0; 8952 8953 err_rif_vrrp_add: 8954 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8955 mlxsw_sp_fid_index(rif->fid), false); 8956 return err; 8957 } 8958 8959 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8960 const struct net_device *macvlan_dev) 8961 { 8962 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8963 struct mlxsw_sp_rif *rif; 8964 8965 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8966 /* If we do not have a RIF, then we already took care of 8967 * removing the macvlan's MAC during RIF deletion. 8968 */ 8969 if (!rif) 8970 return; 8971 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8972 false); 8973 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8974 mlxsw_sp_fid_index(rif->fid), false); 8975 } 8976 8977 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8978 const struct net_device *macvlan_dev) 8979 { 8980 mutex_lock(&mlxsw_sp->router->lock); 8981 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8982 mutex_unlock(&mlxsw_sp->router->lock); 8983 } 8984 8985 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8986 struct net_device *macvlan_dev, 8987 unsigned long event, 8988 struct netlink_ext_ack *extack) 8989 { 8990 switch (event) { 8991 case NETDEV_UP: 8992 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8993 case NETDEV_DOWN: 8994 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8995 break; 8996 } 8997 8998 return 0; 8999 } 9000 9001 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 9002 struct net_device *dev, 9003 unsigned long event, 9004 struct netlink_ext_ack *extack) 9005 { 9006 if (mlxsw_sp_port_dev_check(dev)) 9007 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 9008 else if (netif_is_lag_master(dev)) 9009 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 9010 else if (netif_is_bridge_master(dev)) 9011 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 9012 extack); 9013 else if (is_vlan_dev(dev)) 9014 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 9015 extack); 9016 else if (netif_is_macvlan(dev)) 9017 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 9018 extack); 9019 else 9020 return 0; 9021 } 9022 9023 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 9024 unsigned long event, void *ptr) 9025 { 9026 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 9027 struct net_device *dev = ifa->ifa_dev->dev; 9028 struct mlxsw_sp_router *router; 9029 struct mlxsw_sp_rif *rif; 9030 int err = 0; 9031 9032 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 9033 if (event == NETDEV_UP) 9034 return NOTIFY_DONE; 9035 9036 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 9037 mutex_lock(&router->lock); 9038 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 9039 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9040 goto out; 9041 9042 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 9043 out: 9044 mutex_unlock(&router->lock); 9045 return notifier_from_errno(err); 9046 } 9047 9048 static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 9049 unsigned long event, void *ptr) 9050 { 9051 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 9052 struct net_device *dev = ivi->ivi_dev->dev; 9053 struct mlxsw_sp *mlxsw_sp; 9054 struct mlxsw_sp_rif *rif; 9055 int err = 0; 9056 9057 mlxsw_sp = mlxsw_sp_lower_get(dev); 9058 if (!mlxsw_sp) 9059 return NOTIFY_DONE; 9060 9061 mutex_lock(&mlxsw_sp->router->lock); 9062 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9063 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9064 goto out; 9065 9066 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 9067 out: 9068 mutex_unlock(&mlxsw_sp->router->lock); 9069 return notifier_from_errno(err); 9070 } 9071 9072 struct mlxsw_sp_inet6addr_event_work { 9073 struct work_struct work; 9074 struct mlxsw_sp *mlxsw_sp; 9075 struct net_device *dev; 9076 unsigned long event; 9077 }; 9078 9079 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 9080 { 9081 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 9082 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 9083 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 9084 struct net_device *dev = inet6addr_work->dev; 9085 unsigned long event = inet6addr_work->event; 9086 struct mlxsw_sp_rif *rif; 9087 9088 rtnl_lock(); 9089 mutex_lock(&mlxsw_sp->router->lock); 9090 9091 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9092 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9093 goto out; 9094 9095 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 9096 out: 9097 mutex_unlock(&mlxsw_sp->router->lock); 9098 rtnl_unlock(); 9099 dev_put(dev); 9100 kfree(inet6addr_work); 9101 } 9102 9103 /* Called with rcu_read_lock() */ 9104 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 9105 unsigned long event, void *ptr) 9106 { 9107 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 9108 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 9109 struct net_device *dev = if6->idev->dev; 9110 struct mlxsw_sp_router *router; 9111 9112 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 9113 if (event == NETDEV_UP) 9114 return NOTIFY_DONE; 9115 9116 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 9117 if (!inet6addr_work) 9118 return NOTIFY_BAD; 9119 9120 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 9121 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 9122 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 9123 inet6addr_work->dev = dev; 9124 inet6addr_work->event = event; 9125 dev_hold(dev); 9126 mlxsw_core_schedule_work(&inet6addr_work->work); 9127 9128 return NOTIFY_DONE; 9129 } 9130 9131 static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 9132 unsigned long event, void *ptr) 9133 { 9134 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 9135 struct net_device *dev = i6vi->i6vi_dev->dev; 9136 struct mlxsw_sp *mlxsw_sp; 9137 struct mlxsw_sp_rif *rif; 9138 int err = 0; 9139 9140 mlxsw_sp = mlxsw_sp_lower_get(dev); 9141 if (!mlxsw_sp) 9142 return NOTIFY_DONE; 9143 9144 mutex_lock(&mlxsw_sp->router->lock); 9145 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9146 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 9147 goto out; 9148 9149 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 9150 out: 9151 mutex_unlock(&mlxsw_sp->router->lock); 9152 return notifier_from_errno(err); 9153 } 9154 9155 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 9156 const char *mac, int mtu, u8 mac_profile) 9157 { 9158 char ritr_pl[MLXSW_REG_RITR_LEN]; 9159 int err; 9160 9161 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 9162 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9163 if (err) 9164 return err; 9165 9166 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 9167 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 9168 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 9169 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 9170 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9171 } 9172 9173 static int 9174 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 9175 struct mlxsw_sp_rif *rif, 9176 struct netlink_ext_ack *extack) 9177 { 9178 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9179 u8 old_mac_profile; 9180 u16 fid_index; 9181 int err; 9182 9183 fid_index = mlxsw_sp_fid_index(rif->fid); 9184 9185 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 9186 if (err) 9187 return err; 9188 9189 old_mac_profile = rif->mac_profile_id; 9190 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 9191 extack); 9192 if (err) 9193 goto err_rif_mac_profile_replace; 9194 9195 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 9196 dev->mtu, rif->mac_profile_id); 9197 if (err) 9198 goto err_rif_edit; 9199 9200 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9201 if (err) 9202 goto err_rif_fdb_op; 9203 9204 if (rif->mtu != dev->mtu) { 9205 struct mlxsw_sp_vr *vr; 9206 int i; 9207 9208 /* The RIF is relevant only to its mr_table instance, as unlike 9209 * unicast routing, in multicast routing a RIF cannot be shared 9210 * between several multicast routing tables. 9211 */ 9212 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9213 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9214 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9215 rif, dev->mtu); 9216 } 9217 9218 ether_addr_copy(rif->addr, dev->dev_addr); 9219 rif->mtu = dev->mtu; 9220 9221 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9222 9223 return 0; 9224 9225 err_rif_fdb_op: 9226 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9227 old_mac_profile); 9228 err_rif_edit: 9229 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9230 err_rif_mac_profile_replace: 9231 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9232 return err; 9233 } 9234 9235 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9236 struct netdev_notifier_pre_changeaddr_info *info) 9237 { 9238 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9239 struct mlxsw_sp_rif_mac_profile *profile; 9240 struct netlink_ext_ack *extack; 9241 u8 max_rif_mac_profiles; 9242 u64 occ; 9243 9244 extack = netdev_notifier_info_to_extack(&info->info); 9245 9246 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9247 if (profile) 9248 return 0; 9249 9250 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9251 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9252 if (occ < max_rif_mac_profiles) 9253 return 0; 9254 9255 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9256 return 0; 9257 9258 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9259 return -ENOBUFS; 9260 } 9261 9262 static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp, 9263 struct net_device *dev) 9264 { 9265 struct vlan_dev_priv *vlan; 9266 9267 if (netif_is_lag_master(dev) || 9268 netif_is_bridge_master(dev) || 9269 mlxsw_sp_port_dev_check(dev) || 9270 mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) || 9271 netif_is_l3_master(dev)) 9272 return true; 9273 9274 if (!is_vlan_dev(dev)) 9275 return false; 9276 9277 vlan = vlan_dev_priv(dev); 9278 return netif_is_lag_master(vlan->real_dev) || 9279 netif_is_bridge_master(vlan->real_dev) || 9280 mlxsw_sp_port_dev_check(vlan->real_dev); 9281 } 9282 9283 static struct mlxsw_sp_crif * 9284 mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev) 9285 { 9286 struct mlxsw_sp_crif *crif; 9287 int err; 9288 9289 if (WARN_ON(mlxsw_sp_crif_lookup(router, dev))) 9290 return NULL; 9291 9292 crif = mlxsw_sp_crif_alloc(dev); 9293 if (!crif) 9294 return ERR_PTR(-ENOMEM); 9295 9296 err = mlxsw_sp_crif_insert(router, crif); 9297 if (err) 9298 goto err_netdev_insert; 9299 9300 return crif; 9301 9302 err_netdev_insert: 9303 mlxsw_sp_crif_free(crif); 9304 return ERR_PTR(err); 9305 } 9306 9307 static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router, 9308 struct mlxsw_sp_crif *crif) 9309 { 9310 struct mlxsw_sp_nexthop *nh, *tmp; 9311 9312 mlxsw_sp_crif_remove(router, crif); 9313 9314 list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node) 9315 mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh); 9316 9317 if (crif->rif) 9318 crif->can_destroy = true; 9319 else 9320 mlxsw_sp_crif_free(crif); 9321 } 9322 9323 static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router, 9324 struct net_device *dev) 9325 { 9326 struct mlxsw_sp_crif *crif; 9327 9328 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9329 return 0; 9330 9331 crif = mlxsw_sp_crif_register(router, dev); 9332 return PTR_ERR_OR_ZERO(crif); 9333 } 9334 9335 static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router, 9336 struct net_device *dev) 9337 { 9338 struct mlxsw_sp_crif *crif; 9339 9340 if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev)) 9341 return; 9342 9343 /* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts 9344 * the NETDEV_UNREGISTER message, so we can get here twice. If that's 9345 * what happened, the netdevice state is NETREG_UNREGISTERED. In that 9346 * case, we expect to have collected the CRIF already, and warn if it 9347 * still exists. Otherwise we expect the CRIF to exist. 9348 */ 9349 crif = mlxsw_sp_crif_lookup(router, dev); 9350 if (dev->reg_state == NETREG_UNREGISTERED) { 9351 if (!WARN_ON(crif)) 9352 return; 9353 } 9354 if (WARN_ON(!crif)) 9355 return; 9356 9357 mlxsw_sp_crif_unregister(router, crif); 9358 } 9359 9360 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9361 { 9362 switch (event) { 9363 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9364 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9365 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9366 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9367 return true; 9368 } 9369 9370 return false; 9371 } 9372 9373 static int 9374 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9375 unsigned long event, 9376 struct netdev_notifier_offload_xstats_info *info) 9377 { 9378 switch (info->type) { 9379 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9380 break; 9381 default: 9382 return 0; 9383 } 9384 9385 switch (event) { 9386 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9387 return mlxsw_sp_router_port_l3_stats_enable(rif); 9388 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9389 mlxsw_sp_router_port_l3_stats_disable(rif); 9390 return 0; 9391 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9392 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9393 return 0; 9394 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9395 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9396 } 9397 9398 WARN_ON_ONCE(1); 9399 return 0; 9400 } 9401 9402 static int 9403 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9404 struct net_device *dev, 9405 unsigned long event, 9406 struct netdev_notifier_offload_xstats_info *info) 9407 { 9408 struct mlxsw_sp_rif *rif; 9409 9410 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9411 if (!rif) 9412 return 0; 9413 9414 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9415 } 9416 9417 static bool mlxsw_sp_is_router_event(unsigned long event) 9418 { 9419 switch (event) { 9420 case NETDEV_PRE_CHANGEADDR: 9421 case NETDEV_CHANGEADDR: 9422 case NETDEV_CHANGEMTU: 9423 return true; 9424 default: 9425 return false; 9426 } 9427 } 9428 9429 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9430 unsigned long event, void *ptr) 9431 { 9432 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9433 struct mlxsw_sp *mlxsw_sp; 9434 struct mlxsw_sp_rif *rif; 9435 9436 mlxsw_sp = mlxsw_sp_lower_get(dev); 9437 if (!mlxsw_sp) 9438 return 0; 9439 9440 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9441 if (!rif) 9442 return 0; 9443 9444 switch (event) { 9445 case NETDEV_CHANGEMTU: 9446 case NETDEV_CHANGEADDR: 9447 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9448 case NETDEV_PRE_CHANGEADDR: 9449 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9450 default: 9451 WARN_ON_ONCE(1); 9452 break; 9453 } 9454 9455 return 0; 9456 } 9457 9458 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9459 struct net_device *l3_dev, 9460 struct netlink_ext_ack *extack) 9461 { 9462 struct mlxsw_sp_rif *rif; 9463 9464 /* If netdev is already associated with a RIF, then we need to 9465 * destroy it and create a new one with the new virtual router ID. 9466 */ 9467 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9468 if (rif) 9469 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 9470 extack); 9471 9472 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 9473 } 9474 9475 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9476 struct net_device *l3_dev) 9477 { 9478 struct mlxsw_sp_rif *rif; 9479 9480 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9481 if (!rif) 9482 return; 9483 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 9484 } 9485 9486 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9487 { 9488 struct netdev_notifier_changeupper_info *info = ptr; 9489 9490 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9491 return false; 9492 return netif_is_l3_master(info->upper_dev); 9493 } 9494 9495 static int 9496 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9497 struct netdev_notifier_changeupper_info *info) 9498 { 9499 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9500 int err = 0; 9501 9502 /* We do not create a RIF for a macvlan, but only use it to 9503 * direct more MAC addresses to the router. 9504 */ 9505 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9506 return 0; 9507 9508 switch (event) { 9509 case NETDEV_PRECHANGEUPPER: 9510 break; 9511 case NETDEV_CHANGEUPPER: 9512 if (info->linking) { 9513 struct netlink_ext_ack *extack; 9514 9515 extack = netdev_notifier_info_to_extack(&info->info); 9516 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9517 } else { 9518 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9519 } 9520 break; 9521 } 9522 9523 return err; 9524 } 9525 9526 static int 9527 mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port, 9528 u16 vid, struct net_device *dev, 9529 struct netlink_ext_ack *extack) 9530 { 9531 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 9532 9533 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, 9534 vid); 9535 if (WARN_ON(!mlxsw_sp_port_vlan)) 9536 return -EINVAL; 9537 9538 return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan, 9539 dev, extack); 9540 } 9541 9542 static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9543 struct net_device *lag_dev, 9544 struct netlink_ext_ack *extack) 9545 { 9546 u16 default_vid = MLXSW_SP_DEFAULT_VID; 9547 9548 return mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, 9549 default_vid, lag_dev, 9550 extack); 9551 } 9552 9553 int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port, 9554 struct net_device *lag_dev, 9555 struct netlink_ext_ack *extack) 9556 { 9557 int err; 9558 9559 mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock); 9560 err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack); 9561 mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock); 9562 9563 return err; 9564 } 9565 9566 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 9567 unsigned long event, void *ptr) 9568 { 9569 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 9570 struct mlxsw_sp_router *router; 9571 struct mlxsw_sp *mlxsw_sp; 9572 int err = 0; 9573 9574 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 9575 mlxsw_sp = router->mlxsw_sp; 9576 9577 mutex_lock(&mlxsw_sp->router->lock); 9578 9579 if (event == NETDEV_REGISTER) { 9580 err = mlxsw_sp_netdevice_register(router, dev); 9581 if (err) 9582 /* No need to roll this back, UNREGISTER will collect it 9583 * anyhow. 9584 */ 9585 goto out; 9586 } 9587 9588 if (mlxsw_sp_is_offload_xstats_event(event)) 9589 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 9590 event, ptr); 9591 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 9592 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 9593 event, ptr); 9594 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 9595 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 9596 event, ptr); 9597 else if (mlxsw_sp_is_router_event(event)) 9598 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 9599 else if (mlxsw_sp_is_vrf_event(event, ptr)) 9600 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 9601 9602 if (event == NETDEV_UNREGISTER) 9603 mlxsw_sp_netdevice_unregister(router, dev); 9604 9605 out: 9606 mutex_unlock(&mlxsw_sp->router->lock); 9607 9608 return notifier_from_errno(err); 9609 } 9610 9611 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 9612 struct netdev_nested_priv *priv) 9613 { 9614 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 9615 9616 if (!netif_is_macvlan(dev)) 9617 return 0; 9618 9619 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9620 mlxsw_sp_fid_index(rif->fid), false); 9621 } 9622 9623 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 9624 { 9625 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9626 struct netdev_nested_priv priv = { 9627 .data = (void *)rif, 9628 }; 9629 9630 if (!netif_is_macvlan_port(dev)) 9631 return 0; 9632 9633 netdev_warn(dev, "Router interface is deleted. Upper macvlans will not work\n"); 9634 return netdev_walk_all_upper_dev_rcu(dev, 9635 __mlxsw_sp_rif_macvlan_flush, &priv); 9636 } 9637 9638 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 9639 const struct mlxsw_sp_rif_params *params) 9640 { 9641 struct mlxsw_sp_rif_subport *rif_subport; 9642 9643 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9644 refcount_set(&rif_subport->ref_count, 1); 9645 rif_subport->vid = params->vid; 9646 rif_subport->lag = params->lag; 9647 if (params->lag) 9648 rif_subport->lag_id = params->lag_id; 9649 else 9650 rif_subport->system_port = params->system_port; 9651 } 9652 9653 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 9654 { 9655 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9656 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9657 struct mlxsw_sp_rif_subport *rif_subport; 9658 char ritr_pl[MLXSW_REG_RITR_LEN]; 9659 u16 efid; 9660 9661 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9662 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 9663 rif->rif_index, rif->vr_id, dev->mtu); 9664 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 9665 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9666 efid = mlxsw_sp_fid_index(rif->fid); 9667 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 9668 rif_subport->lag ? rif_subport->lag_id : 9669 rif_subport->system_port, 9670 efid, 0); 9671 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9672 } 9673 9674 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 9675 struct netlink_ext_ack *extack) 9676 { 9677 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9678 u8 mac_profile; 9679 int err; 9680 9681 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 9682 &mac_profile, extack); 9683 if (err) 9684 return err; 9685 rif->mac_profile_id = mac_profile; 9686 9687 err = mlxsw_sp_rif_subport_op(rif, true); 9688 if (err) 9689 goto err_rif_subport_op; 9690 9691 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9692 mlxsw_sp_fid_index(rif->fid), true); 9693 if (err) 9694 goto err_rif_fdb_op; 9695 9696 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 9697 if (err) 9698 goto err_fid_rif_set; 9699 9700 return 0; 9701 9702 err_fid_rif_set: 9703 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9704 mlxsw_sp_fid_index(rif->fid), false); 9705 err_rif_fdb_op: 9706 mlxsw_sp_rif_subport_op(rif, false); 9707 err_rif_subport_op: 9708 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 9709 return err; 9710 } 9711 9712 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 9713 { 9714 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9715 struct mlxsw_sp_fid *fid = rif->fid; 9716 9717 mlxsw_sp_fid_rif_unset(fid); 9718 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9719 mlxsw_sp_fid_index(fid), false); 9720 mlxsw_sp_rif_macvlan_flush(rif); 9721 mlxsw_sp_rif_subport_op(rif, false); 9722 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9723 } 9724 9725 static struct mlxsw_sp_fid * 9726 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 9727 struct netlink_ext_ack *extack) 9728 { 9729 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 9730 } 9731 9732 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 9733 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 9734 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 9735 .setup = mlxsw_sp_rif_subport_setup, 9736 .configure = mlxsw_sp_rif_subport_configure, 9737 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 9738 .fid_get = mlxsw_sp_rif_subport_fid_get, 9739 }; 9740 9741 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable) 9742 { 9743 enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF; 9744 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9745 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9746 char ritr_pl[MLXSW_REG_RITR_LEN]; 9747 9748 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9749 dev->mtu); 9750 mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr); 9751 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9752 mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid); 9753 9754 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9755 } 9756 9757 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9758 { 9759 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9760 } 9761 9762 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 9763 struct netlink_ext_ack *extack) 9764 { 9765 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9766 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9767 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9768 u8 mac_profile; 9769 int err; 9770 9771 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 9772 &mac_profile, extack); 9773 if (err) 9774 return err; 9775 rif->mac_profile_id = mac_profile; 9776 9777 err = mlxsw_sp_rif_fid_op(rif, fid_index, true); 9778 if (err) 9779 goto err_rif_fid_op; 9780 9781 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9782 mlxsw_sp_router_port(mlxsw_sp), true); 9783 if (err) 9784 goto err_fid_mc_flood_set; 9785 9786 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9787 mlxsw_sp_router_port(mlxsw_sp), true); 9788 if (err) 9789 goto err_fid_bc_flood_set; 9790 9791 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9792 mlxsw_sp_fid_index(rif->fid), true); 9793 if (err) 9794 goto err_rif_fdb_op; 9795 9796 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 9797 if (err) 9798 goto err_fid_rif_set; 9799 9800 return 0; 9801 9802 err_fid_rif_set: 9803 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9804 mlxsw_sp_fid_index(rif->fid), false); 9805 err_rif_fdb_op: 9806 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9807 mlxsw_sp_router_port(mlxsw_sp), false); 9808 err_fid_bc_flood_set: 9809 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9810 mlxsw_sp_router_port(mlxsw_sp), false); 9811 err_fid_mc_flood_set: 9812 mlxsw_sp_rif_fid_op(rif, fid_index, false); 9813 err_rif_fid_op: 9814 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 9815 return err; 9816 } 9817 9818 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9819 { 9820 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9821 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9822 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9823 struct mlxsw_sp_fid *fid = rif->fid; 9824 9825 mlxsw_sp_fid_rif_unset(fid); 9826 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9827 mlxsw_sp_fid_index(fid), false); 9828 mlxsw_sp_rif_macvlan_flush(rif); 9829 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9830 mlxsw_sp_router_port(mlxsw_sp), false); 9831 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9832 mlxsw_sp_router_port(mlxsw_sp), false); 9833 mlxsw_sp_rif_fid_op(rif, fid_index, false); 9834 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9835 } 9836 9837 static struct mlxsw_sp_fid * 9838 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9839 struct netlink_ext_ack *extack) 9840 { 9841 int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif); 9842 9843 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex); 9844 } 9845 9846 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9847 { 9848 struct switchdev_notifier_fdb_info info = {}; 9849 struct net_device *dev; 9850 9851 dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0); 9852 if (!dev) 9853 return; 9854 9855 info.addr = mac; 9856 info.vid = 0; 9857 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9858 NULL); 9859 } 9860 9861 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9862 .type = MLXSW_SP_RIF_TYPE_FID, 9863 .rif_size = sizeof(struct mlxsw_sp_rif), 9864 .configure = mlxsw_sp_rif_fid_configure, 9865 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9866 .fid_get = mlxsw_sp_rif_fid_fid_get, 9867 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9868 }; 9869 9870 static struct mlxsw_sp_fid * 9871 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9872 struct netlink_ext_ack *extack) 9873 { 9874 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9875 struct net_device *br_dev; 9876 u16 vid; 9877 int err; 9878 9879 if (is_vlan_dev(dev)) { 9880 vid = vlan_dev_vlan_id(dev); 9881 br_dev = vlan_dev_real_dev(dev); 9882 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9883 return ERR_PTR(-EINVAL); 9884 } else { 9885 err = br_vlan_get_pvid(dev, &vid); 9886 if (err < 0 || !vid) { 9887 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9888 return ERR_PTR(-EINVAL); 9889 } 9890 } 9891 9892 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9893 } 9894 9895 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9896 { 9897 struct net_device *rif_dev = mlxsw_sp_rif_dev(rif); 9898 struct switchdev_notifier_fdb_info info = {}; 9899 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9900 struct net_device *br_dev; 9901 struct net_device *dev; 9902 9903 br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev; 9904 dev = br_fdb_find_port(br_dev, mac, vid); 9905 if (!dev) 9906 return; 9907 9908 info.addr = mac; 9909 info.vid = vid; 9910 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9911 NULL); 9912 } 9913 9914 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid, 9915 bool enable) 9916 { 9917 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9918 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9919 char ritr_pl[MLXSW_REG_RITR_LEN]; 9920 9921 mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id, 9922 dev->mtu, dev->dev_addr, 9923 rif->mac_profile_id, vid, efid); 9924 9925 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9926 } 9927 9928 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid, 9929 struct netlink_ext_ack *extack) 9930 { 9931 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9932 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9933 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9934 u8 mac_profile; 9935 int err; 9936 9937 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 9938 &mac_profile, extack); 9939 if (err) 9940 return err; 9941 rif->mac_profile_id = mac_profile; 9942 9943 err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true); 9944 if (err) 9945 goto err_rif_vlan_fid_op; 9946 9947 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9948 mlxsw_sp_router_port(mlxsw_sp), true); 9949 if (err) 9950 goto err_fid_mc_flood_set; 9951 9952 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9953 mlxsw_sp_router_port(mlxsw_sp), true); 9954 if (err) 9955 goto err_fid_bc_flood_set; 9956 9957 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9958 mlxsw_sp_fid_index(rif->fid), true); 9959 if (err) 9960 goto err_rif_fdb_op; 9961 9962 err = mlxsw_sp_fid_rif_set(rif->fid, rif); 9963 if (err) 9964 goto err_fid_rif_set; 9965 9966 return 0; 9967 9968 err_fid_rif_set: 9969 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9970 mlxsw_sp_fid_index(rif->fid), false); 9971 err_rif_fdb_op: 9972 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9973 mlxsw_sp_router_port(mlxsw_sp), false); 9974 err_fid_bc_flood_set: 9975 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9976 mlxsw_sp_router_port(mlxsw_sp), false); 9977 err_fid_mc_flood_set: 9978 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 9979 err_rif_vlan_fid_op: 9980 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 9981 return err; 9982 } 9983 9984 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif) 9985 { 9986 struct net_device *dev = mlxsw_sp_rif_dev(rif); 9987 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9988 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9989 9990 mlxsw_sp_fid_rif_unset(rif->fid); 9991 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9992 mlxsw_sp_fid_index(rif->fid), false); 9993 mlxsw_sp_rif_macvlan_flush(rif); 9994 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9995 mlxsw_sp_router_port(mlxsw_sp), false); 9996 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9997 mlxsw_sp_router_port(mlxsw_sp), false); 9998 mlxsw_sp_rif_vlan_op(rif, vid, 0, false); 9999 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 10000 } 10001 10002 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10003 struct netlink_ext_ack *extack) 10004 { 10005 return mlxsw_sp_rif_vlan_configure(rif, 0, extack); 10006 } 10007 10008 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = { 10009 .type = MLXSW_SP_RIF_TYPE_VLAN, 10010 .rif_size = sizeof(struct mlxsw_sp_rif), 10011 .configure = mlxsw_sp1_rif_vlan_configure, 10012 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10013 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10014 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10015 }; 10016 10017 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif, 10018 struct netlink_ext_ack *extack) 10019 { 10020 u16 efid = mlxsw_sp_fid_index(rif->fid); 10021 10022 return mlxsw_sp_rif_vlan_configure(rif, efid, extack); 10023 } 10024 10025 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = { 10026 .type = MLXSW_SP_RIF_TYPE_VLAN, 10027 .rif_size = sizeof(struct mlxsw_sp_rif), 10028 .configure = mlxsw_sp2_rif_vlan_configure, 10029 .deconfigure = mlxsw_sp_rif_vlan_deconfigure, 10030 .fid_get = mlxsw_sp_rif_vlan_fid_get, 10031 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 10032 }; 10033 10034 static struct mlxsw_sp_rif_ipip_lb * 10035 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 10036 { 10037 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 10038 } 10039 10040 static void 10041 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 10042 const struct mlxsw_sp_rif_params *params) 10043 { 10044 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 10045 struct mlxsw_sp_rif_ipip_lb *rif_lb; 10046 10047 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 10048 common); 10049 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 10050 rif_lb->lb_config = params_lb->lb_config; 10051 } 10052 10053 static int 10054 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10055 struct netlink_ext_ack *extack) 10056 { 10057 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10058 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10059 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10060 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10061 struct mlxsw_sp_vr *ul_vr; 10062 int err; 10063 10064 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack); 10065 if (IS_ERR(ul_vr)) 10066 return PTR_ERR(ul_vr); 10067 10068 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 10069 if (err) 10070 goto err_loopback_op; 10071 10072 lb_rif->ul_vr_id = ul_vr->id; 10073 lb_rif->ul_rif_id = 0; 10074 ++ul_vr->rif_count; 10075 return 0; 10076 10077 err_loopback_op: 10078 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10079 return err; 10080 } 10081 10082 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10083 { 10084 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10085 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10086 struct mlxsw_sp_vr *ul_vr; 10087 10088 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 10089 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 10090 10091 --ul_vr->rif_count; 10092 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 10093 } 10094 10095 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 10096 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10097 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10098 .setup = mlxsw_sp_rif_ipip_lb_setup, 10099 .configure = mlxsw_sp1_rif_ipip_lb_configure, 10100 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 10101 }; 10102 10103 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 10104 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10105 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp1_rif_vlan_ops, 10106 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10107 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 10108 }; 10109 10110 static int 10111 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 10112 { 10113 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10114 char ritr_pl[MLXSW_REG_RITR_LEN]; 10115 10116 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 10117 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 10118 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 10119 MLXSW_REG_RITR_LOOPBACK_GENERIC); 10120 10121 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 10122 } 10123 10124 static struct mlxsw_sp_rif * 10125 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 10126 struct mlxsw_sp_crif *ul_crif, 10127 struct netlink_ext_ack *extack) 10128 { 10129 struct mlxsw_sp_rif *ul_rif; 10130 u8 rif_entries = 1; 10131 u16 rif_index; 10132 int err; 10133 10134 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries); 10135 if (err) { 10136 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 10137 return ERR_PTR(err); 10138 } 10139 10140 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, 10141 ul_crif); 10142 if (!ul_rif) { 10143 err = -ENOMEM; 10144 goto err_rif_alloc; 10145 } 10146 10147 mlxsw_sp->router->rifs[rif_index] = ul_rif; 10148 ul_rif->mlxsw_sp = mlxsw_sp; 10149 ul_rif->rif_entries = rif_entries; 10150 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 10151 if (err) 10152 goto ul_rif_op_err; 10153 10154 atomic_add(rif_entries, &mlxsw_sp->router->rifs_count); 10155 return ul_rif; 10156 10157 ul_rif_op_err: 10158 mlxsw_sp->router->rifs[rif_index] = NULL; 10159 mlxsw_sp_rif_free(ul_rif); 10160 err_rif_alloc: 10161 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10162 return ERR_PTR(err); 10163 } 10164 10165 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 10166 { 10167 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10168 u8 rif_entries = ul_rif->rif_entries; 10169 u16 rif_index = ul_rif->rif_index; 10170 10171 atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count); 10172 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 10173 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 10174 mlxsw_sp_rif_free(ul_rif); 10175 mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries); 10176 } 10177 10178 static struct mlxsw_sp_rif * 10179 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 10180 struct mlxsw_sp_crif *ul_crif, 10181 struct netlink_ext_ack *extack) 10182 { 10183 struct mlxsw_sp_vr *vr; 10184 int err; 10185 10186 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 10187 if (IS_ERR(vr)) 10188 return ERR_CAST(vr); 10189 10190 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 10191 return vr->ul_rif; 10192 10193 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack); 10194 if (IS_ERR(vr->ul_rif)) { 10195 err = PTR_ERR(vr->ul_rif); 10196 goto err_ul_rif_create; 10197 } 10198 10199 vr->rif_count++; 10200 refcount_set(&vr->ul_rif_refcnt, 1); 10201 10202 return vr->ul_rif; 10203 10204 err_ul_rif_create: 10205 mlxsw_sp_vr_put(mlxsw_sp, vr); 10206 return ERR_PTR(err); 10207 } 10208 10209 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 10210 { 10211 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 10212 struct mlxsw_sp_vr *vr; 10213 10214 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 10215 10216 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 10217 return; 10218 10219 vr->rif_count--; 10220 mlxsw_sp_ul_rif_destroy(ul_rif); 10221 mlxsw_sp_vr_put(mlxsw_sp, vr); 10222 } 10223 10224 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 10225 u16 *ul_rif_index) 10226 { 10227 struct mlxsw_sp_rif *ul_rif; 10228 int err = 0; 10229 10230 mutex_lock(&mlxsw_sp->router->lock); 10231 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL); 10232 if (IS_ERR(ul_rif)) { 10233 err = PTR_ERR(ul_rif); 10234 goto out; 10235 } 10236 *ul_rif_index = ul_rif->rif_index; 10237 out: 10238 mutex_unlock(&mlxsw_sp->router->lock); 10239 return err; 10240 } 10241 10242 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 10243 { 10244 struct mlxsw_sp_rif *ul_rif; 10245 10246 mutex_lock(&mlxsw_sp->router->lock); 10247 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 10248 if (WARN_ON(!ul_rif)) 10249 goto out; 10250 10251 mlxsw_sp_ul_rif_put(ul_rif); 10252 out: 10253 mutex_unlock(&mlxsw_sp->router->lock); 10254 } 10255 10256 static int 10257 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 10258 struct netlink_ext_ack *extack) 10259 { 10260 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10261 struct net_device *dev = mlxsw_sp_rif_dev(rif); 10262 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev); 10263 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10264 struct mlxsw_sp_rif *ul_rif; 10265 int err; 10266 10267 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack); 10268 if (IS_ERR(ul_rif)) 10269 return PTR_ERR(ul_rif); 10270 10271 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 10272 if (err) 10273 goto err_loopback_op; 10274 10275 lb_rif->ul_vr_id = 0; 10276 lb_rif->ul_rif_id = ul_rif->rif_index; 10277 10278 return 0; 10279 10280 err_loopback_op: 10281 mlxsw_sp_ul_rif_put(ul_rif); 10282 return err; 10283 } 10284 10285 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 10286 { 10287 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 10288 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 10289 struct mlxsw_sp_rif *ul_rif; 10290 10291 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 10292 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 10293 mlxsw_sp_ul_rif_put(ul_rif); 10294 } 10295 10296 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 10297 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 10298 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 10299 .setup = mlxsw_sp_rif_ipip_lb_setup, 10300 .configure = mlxsw_sp2_rif_ipip_lb_configure, 10301 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 10302 }; 10303 10304 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 10305 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 10306 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp2_rif_vlan_ops, 10307 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 10308 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 10309 }; 10310 10311 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp) 10312 { 10313 struct gen_pool *rifs_table; 10314 int err; 10315 10316 rifs_table = gen_pool_create(0, -1); 10317 if (!rifs_table) 10318 return -ENOMEM; 10319 10320 gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align, 10321 NULL); 10322 10323 err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET, 10324 MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1); 10325 if (err) 10326 goto err_gen_pool_add; 10327 10328 mlxsw_sp->router->rifs_table = rifs_table; 10329 10330 return 0; 10331 10332 err_gen_pool_add: 10333 gen_pool_destroy(rifs_table); 10334 return err; 10335 } 10336 10337 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp) 10338 { 10339 gen_pool_destroy(mlxsw_sp->router->rifs_table); 10340 } 10341 10342 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 10343 { 10344 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10345 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10346 struct mlxsw_core *core = mlxsw_sp->core; 10347 int err; 10348 10349 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 10350 return -EIO; 10351 mlxsw_sp->router->max_rif_mac_profile = 10352 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 10353 10354 mlxsw_sp->router->rifs = kcalloc(max_rifs, 10355 sizeof(struct mlxsw_sp_rif *), 10356 GFP_KERNEL); 10357 if (!mlxsw_sp->router->rifs) 10358 return -ENOMEM; 10359 10360 err = mlxsw_sp_rifs_table_init(mlxsw_sp); 10361 if (err) 10362 goto err_rifs_table_init; 10363 10364 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 10365 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 10366 atomic_set(&mlxsw_sp->router->rifs_count, 0); 10367 devl_resource_occ_get_register(devlink, 10368 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 10369 mlxsw_sp_rif_mac_profiles_occ_get, 10370 mlxsw_sp); 10371 devl_resource_occ_get_register(devlink, 10372 MLXSW_SP_RESOURCE_RIFS, 10373 mlxsw_sp_rifs_occ_get, 10374 mlxsw_sp); 10375 10376 return 0; 10377 10378 err_rifs_table_init: 10379 kfree(mlxsw_sp->router->rifs); 10380 return err; 10381 } 10382 10383 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 10384 { 10385 int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10386 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 10387 int i; 10388 10389 WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count)); 10390 for (i = 0; i < max_rifs; i++) 10391 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 10392 10393 devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); 10394 devl_resource_occ_get_unregister(devlink, 10395 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 10396 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 10397 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 10398 mlxsw_sp_rifs_table_fini(mlxsw_sp); 10399 kfree(mlxsw_sp->router->rifs); 10400 } 10401 10402 static int 10403 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 10404 { 10405 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 10406 10407 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 10408 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 10409 } 10410 10411 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 10412 { 10413 int err; 10414 10415 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 10416 10417 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 10418 if (err) 10419 return err; 10420 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 10421 if (err) 10422 return err; 10423 10424 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 10425 } 10426 10427 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 10428 { 10429 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 10430 return mlxsw_sp_ipips_init(mlxsw_sp); 10431 } 10432 10433 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 10434 { 10435 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 10436 return mlxsw_sp_ipips_init(mlxsw_sp); 10437 } 10438 10439 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 10440 { 10441 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 10442 } 10443 10444 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 10445 { 10446 struct mlxsw_sp_router *router; 10447 10448 /* Flush pending FIB notifications and then flush the device's 10449 * table before requesting another dump. The FIB notification 10450 * block is unregistered, so no need to take RTNL. 10451 */ 10452 mlxsw_core_flush_owq(); 10453 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 10454 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 10455 } 10456 10457 #ifdef CONFIG_IP_ROUTE_MULTIPATH 10458 struct mlxsw_sp_mp_hash_config { 10459 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 10460 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 10461 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 10462 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 10463 bool inc_parsing_depth; 10464 }; 10465 10466 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 10467 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 10468 10469 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 10470 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 10471 10472 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 10473 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 10474 10475 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 10476 { 10477 unsigned long *inner_headers = config->inner_headers; 10478 unsigned long *inner_fields = config->inner_fields; 10479 10480 /* IPv4 inner */ 10481 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10482 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10483 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10484 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10485 /* IPv6 inner */ 10486 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10487 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10488 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10489 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10490 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10491 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10492 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10493 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10494 } 10495 10496 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10497 { 10498 unsigned long *headers = config->headers; 10499 unsigned long *fields = config->fields; 10500 10501 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10502 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10503 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10504 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10505 } 10506 10507 static void 10508 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 10509 u32 hash_fields) 10510 { 10511 unsigned long *inner_headers = config->inner_headers; 10512 unsigned long *inner_fields = config->inner_fields; 10513 10514 /* IPv4 Inner */ 10515 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 10516 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 10517 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 10518 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 10519 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 10520 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 10521 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10522 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 10523 /* IPv6 inner */ 10524 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 10525 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 10526 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 10527 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 10528 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 10529 } 10530 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 10531 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 10532 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 10533 } 10534 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 10535 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 10536 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 10537 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 10538 /* L4 inner */ 10539 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 10540 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 10541 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 10542 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 10543 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 10544 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 10545 } 10546 10547 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 10548 struct mlxsw_sp_mp_hash_config *config) 10549 { 10550 struct net *net = mlxsw_sp_net(mlxsw_sp); 10551 unsigned long *headers = config->headers; 10552 unsigned long *fields = config->fields; 10553 u32 hash_fields; 10554 10555 switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) { 10556 case 0: 10557 mlxsw_sp_mp4_hash_outer_addr(config); 10558 break; 10559 case 1: 10560 mlxsw_sp_mp4_hash_outer_addr(config); 10561 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10562 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10563 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10564 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10565 break; 10566 case 2: 10567 /* Outer */ 10568 mlxsw_sp_mp4_hash_outer_addr(config); 10569 /* Inner */ 10570 mlxsw_sp_mp_hash_inner_l3(config); 10571 break; 10572 case 3: 10573 hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields); 10574 /* Outer */ 10575 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10576 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10577 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10578 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 10579 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10580 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 10581 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10582 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10583 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10584 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10585 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10586 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10587 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10588 /* Inner */ 10589 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10590 break; 10591 } 10592 } 10593 10594 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10595 { 10596 unsigned long *headers = config->headers; 10597 unsigned long *fields = config->fields; 10598 10599 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10600 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10601 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10602 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10603 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10604 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10605 } 10606 10607 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 10608 struct mlxsw_sp_mp_hash_config *config) 10609 { 10610 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 10611 unsigned long *headers = config->headers; 10612 unsigned long *fields = config->fields; 10613 10614 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 10615 case 0: 10616 mlxsw_sp_mp6_hash_outer_addr(config); 10617 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10618 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10619 break; 10620 case 1: 10621 mlxsw_sp_mp6_hash_outer_addr(config); 10622 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10623 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10624 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10625 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10626 break; 10627 case 2: 10628 /* Outer */ 10629 mlxsw_sp_mp6_hash_outer_addr(config); 10630 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10631 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10632 /* Inner */ 10633 mlxsw_sp_mp_hash_inner_l3(config); 10634 config->inc_parsing_depth = true; 10635 break; 10636 case 3: 10637 /* Outer */ 10638 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10639 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10640 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10641 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 10642 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10643 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10644 } 10645 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 10646 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10647 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10648 } 10649 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10650 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10651 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 10652 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10653 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10654 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10655 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10656 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10657 /* Inner */ 10658 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10659 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 10660 config->inc_parsing_depth = true; 10661 break; 10662 } 10663 } 10664 10665 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 10666 bool old_inc_parsing_depth, 10667 bool new_inc_parsing_depth) 10668 { 10669 int err; 10670 10671 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 10672 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 10673 if (err) 10674 return err; 10675 mlxsw_sp->router->inc_parsing_depth = true; 10676 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 10677 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 10678 mlxsw_sp->router->inc_parsing_depth = false; 10679 } 10680 10681 return 0; 10682 } 10683 10684 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10685 { 10686 bool old_inc_parsing_depth, new_inc_parsing_depth; 10687 struct mlxsw_sp_mp_hash_config config = {}; 10688 char recr2_pl[MLXSW_REG_RECR2_LEN]; 10689 unsigned long bit; 10690 u32 seed; 10691 int err; 10692 10693 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 10694 mlxsw_reg_recr2_pack(recr2_pl, seed); 10695 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 10696 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 10697 10698 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 10699 new_inc_parsing_depth = config.inc_parsing_depth; 10700 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 10701 old_inc_parsing_depth, 10702 new_inc_parsing_depth); 10703 if (err) 10704 return err; 10705 10706 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 10707 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 10708 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 10709 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 10710 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 10711 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 10712 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 10713 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 10714 10715 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 10716 if (err) 10717 goto err_reg_write; 10718 10719 return 0; 10720 10721 err_reg_write: 10722 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 10723 old_inc_parsing_depth); 10724 return err; 10725 } 10726 10727 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 10728 { 10729 bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 10730 10731 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth, 10732 false); 10733 } 10734 #else 10735 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10736 { 10737 return 0; 10738 } 10739 10740 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp) 10741 { 10742 } 10743 #endif 10744 10745 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 10746 { 10747 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 10748 unsigned int i; 10749 10750 MLXSW_REG_ZERO(rdpm, rdpm_pl); 10751 10752 /* HW is determining switch priority based on DSCP-bits, but the 10753 * kernel is still doing that based on the ToS. Since there's a 10754 * mismatch in bits we need to make sure to translate the right 10755 * value ToS would observe, skipping the 2 least-significant ECN bits. 10756 */ 10757 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 10758 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 10759 10760 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 10761 } 10762 10763 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 10764 { 10765 struct net *net = mlxsw_sp_net(mlxsw_sp); 10766 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10767 u64 max_rifs; 10768 bool usp; 10769 10770 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 10771 return -EIO; 10772 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10773 usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority); 10774 10775 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 10776 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 10777 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 10778 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10779 } 10780 10781 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10782 { 10783 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10784 10785 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 10786 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10787 } 10788 10789 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp, 10790 struct netlink_ext_ack *extack) 10791 { 10792 struct mlxsw_sp_router *router = mlxsw_sp->router; 10793 struct mlxsw_sp_rif *lb_rif; 10794 int err; 10795 10796 router->lb_crif = mlxsw_sp_crif_alloc(NULL); 10797 if (!router->lb_crif) 10798 return -ENOMEM; 10799 10800 /* Create a generic loopback RIF associated with the main table 10801 * (default VRF). Any table can be used, but the main table exists 10802 * anyway, so we do not waste resources. Loopback RIFs are usually 10803 * created with a NULL CRIF, but this RIF is used as a fallback RIF 10804 * for blackhole nexthops, and nexthops expect to have a valid CRIF. 10805 */ 10806 lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif, 10807 extack); 10808 if (IS_ERR(lb_rif)) { 10809 err = PTR_ERR(lb_rif); 10810 goto err_ul_rif_get; 10811 } 10812 10813 return 0; 10814 10815 err_ul_rif_get: 10816 mlxsw_sp_crif_free(router->lb_crif); 10817 return err; 10818 } 10819 10820 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 10821 { 10822 mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif); 10823 mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif); 10824 } 10825 10826 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 10827 { 10828 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 10829 10830 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 10831 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 10832 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10833 10834 return 0; 10835 } 10836 10837 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 10838 .init = mlxsw_sp1_router_init, 10839 .ipips_init = mlxsw_sp1_ipips_init, 10840 }; 10841 10842 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 10843 { 10844 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 10845 10846 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 10847 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 10848 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10849 10850 return 0; 10851 } 10852 10853 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 10854 .init = mlxsw_sp2_router_init, 10855 .ipips_init = mlxsw_sp2_ipips_init, 10856 }; 10857 10858 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 10859 struct netlink_ext_ack *extack) 10860 { 10861 struct mlxsw_sp_router *router; 10862 struct notifier_block *nb; 10863 int err; 10864 10865 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 10866 if (!router) 10867 return -ENOMEM; 10868 mutex_init(&router->lock); 10869 mlxsw_sp->router = router; 10870 router->mlxsw_sp = mlxsw_sp; 10871 10872 err = mlxsw_sp->router_ops->init(mlxsw_sp); 10873 if (err) 10874 goto err_router_ops_init; 10875 10876 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 10877 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 10878 mlxsw_sp_nh_grp_activity_work); 10879 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 10880 err = __mlxsw_sp_router_init(mlxsw_sp); 10881 if (err) 10882 goto err_router_init; 10883 10884 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 10885 if (err) 10886 goto err_ipips_init; 10887 10888 err = rhashtable_init(&mlxsw_sp->router->crif_ht, 10889 &mlxsw_sp_crif_ht_params); 10890 if (err) 10891 goto err_crif_ht_init; 10892 10893 err = mlxsw_sp_rifs_init(mlxsw_sp); 10894 if (err) 10895 goto err_rifs_init; 10896 10897 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 10898 &mlxsw_sp_nexthop_ht_params); 10899 if (err) 10900 goto err_nexthop_ht_init; 10901 10902 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 10903 &mlxsw_sp_nexthop_group_ht_params); 10904 if (err) 10905 goto err_nexthop_group_ht_init; 10906 10907 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 10908 err = mlxsw_sp_lpm_init(mlxsw_sp); 10909 if (err) 10910 goto err_lpm_init; 10911 10912 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 10913 if (err) 10914 goto err_mr_init; 10915 10916 err = mlxsw_sp_vrs_init(mlxsw_sp); 10917 if (err) 10918 goto err_vrs_init; 10919 10920 err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack); 10921 if (err) 10922 goto err_lb_rif_init; 10923 10924 err = mlxsw_sp_neigh_init(mlxsw_sp); 10925 if (err) 10926 goto err_neigh_init; 10927 10928 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 10929 if (err) 10930 goto err_mp_hash_init; 10931 10932 err = mlxsw_sp_dscp_init(mlxsw_sp); 10933 if (err) 10934 goto err_dscp_init; 10935 10936 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 10937 err = register_inetaddr_notifier(&router->inetaddr_nb); 10938 if (err) 10939 goto err_register_inetaddr_notifier; 10940 10941 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 10942 err = register_inet6addr_notifier(&router->inet6addr_nb); 10943 if (err) 10944 goto err_register_inet6addr_notifier; 10945 10946 router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event; 10947 err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 10948 if (err) 10949 goto err_register_inetaddr_valid_notifier; 10950 10951 nb = &router->inet6addr_valid_nb; 10952 nb->notifier_call = mlxsw_sp_inet6addr_valid_event; 10953 err = register_inet6addr_validator_notifier(nb); 10954 if (err) 10955 goto err_register_inet6addr_valid_notifier; 10956 10957 mlxsw_sp->router->netevent_nb.notifier_call = 10958 mlxsw_sp_router_netevent_event; 10959 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10960 if (err) 10961 goto err_register_netevent_notifier; 10962 10963 mlxsw_sp->router->nexthop_nb.notifier_call = 10964 mlxsw_sp_nexthop_obj_event; 10965 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10966 &mlxsw_sp->router->nexthop_nb, 10967 extack); 10968 if (err) 10969 goto err_register_nexthop_notifier; 10970 10971 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 10972 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10973 &mlxsw_sp->router->fib_nb, 10974 mlxsw_sp_router_fib_dump_flush, extack); 10975 if (err) 10976 goto err_register_fib_notifier; 10977 10978 mlxsw_sp->router->netdevice_nb.notifier_call = 10979 mlxsw_sp_router_netdevice_event; 10980 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10981 &mlxsw_sp->router->netdevice_nb); 10982 if (err) 10983 goto err_register_netdev_notifier; 10984 10985 return 0; 10986 10987 err_register_netdev_notifier: 10988 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10989 &mlxsw_sp->router->fib_nb); 10990 err_register_fib_notifier: 10991 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10992 &mlxsw_sp->router->nexthop_nb); 10993 err_register_nexthop_notifier: 10994 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10995 err_register_netevent_notifier: 10996 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 10997 err_register_inet6addr_valid_notifier: 10998 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 10999 err_register_inetaddr_valid_notifier: 11000 unregister_inet6addr_notifier(&router->inet6addr_nb); 11001 err_register_inet6addr_notifier: 11002 unregister_inetaddr_notifier(&router->inetaddr_nb); 11003 err_register_inetaddr_notifier: 11004 mlxsw_core_flush_owq(); 11005 err_dscp_init: 11006 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11007 err_mp_hash_init: 11008 mlxsw_sp_neigh_fini(mlxsw_sp); 11009 err_neigh_init: 11010 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11011 err_lb_rif_init: 11012 mlxsw_sp_vrs_fini(mlxsw_sp); 11013 err_vrs_init: 11014 mlxsw_sp_mr_fini(mlxsw_sp); 11015 err_mr_init: 11016 mlxsw_sp_lpm_fini(mlxsw_sp); 11017 err_lpm_init: 11018 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 11019 err_nexthop_group_ht_init: 11020 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 11021 err_nexthop_ht_init: 11022 mlxsw_sp_rifs_fini(mlxsw_sp); 11023 err_rifs_init: 11024 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11025 err_crif_ht_init: 11026 mlxsw_sp_ipips_fini(mlxsw_sp); 11027 err_ipips_init: 11028 __mlxsw_sp_router_fini(mlxsw_sp); 11029 err_router_init: 11030 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 11031 err_router_ops_init: 11032 mutex_destroy(&mlxsw_sp->router->lock); 11033 kfree(mlxsw_sp->router); 11034 return err; 11035 } 11036 11037 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 11038 { 11039 struct mlxsw_sp_router *router = mlxsw_sp->router; 11040 11041 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 11042 &router->netdevice_nb); 11043 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb); 11044 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 11045 &router->nexthop_nb); 11046 unregister_netevent_notifier(&router->netevent_nb); 11047 unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb); 11048 unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb); 11049 unregister_inet6addr_notifier(&router->inet6addr_nb); 11050 unregister_inetaddr_notifier(&router->inetaddr_nb); 11051 mlxsw_core_flush_owq(); 11052 mlxsw_sp_mp_hash_fini(mlxsw_sp); 11053 mlxsw_sp_neigh_fini(mlxsw_sp); 11054 mlxsw_sp_lb_rif_fini(mlxsw_sp); 11055 mlxsw_sp_vrs_fini(mlxsw_sp); 11056 mlxsw_sp_mr_fini(mlxsw_sp); 11057 mlxsw_sp_lpm_fini(mlxsw_sp); 11058 rhashtable_destroy(&router->nexthop_group_ht); 11059 rhashtable_destroy(&router->nexthop_ht); 11060 mlxsw_sp_rifs_fini(mlxsw_sp); 11061 rhashtable_destroy(&mlxsw_sp->router->crif_ht); 11062 mlxsw_sp_ipips_fini(mlxsw_sp); 11063 __mlxsw_sp_router_fini(mlxsw_sp); 11064 cancel_delayed_work_sync(&router->nh_grp_activity_dw); 11065 mutex_destroy(&router->lock); 11066 kfree(router); 11067 } 11068