1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/rhashtable.h> 7 #include <linux/bitops.h> 8 #include <linux/in6.h> 9 #include <linux/notifier.h> 10 #include <linux/inetdevice.h> 11 #include <linux/netdevice.h> 12 #include <linux/if_bridge.h> 13 #include <linux/socket.h> 14 #include <linux/route.h> 15 #include <linux/gcd.h> 16 #include <linux/if_macvlan.h> 17 #include <linux/refcount.h> 18 #include <linux/jhash.h> 19 #include <linux/net_namespace.h> 20 #include <linux/mutex.h> 21 #include <net/netevent.h> 22 #include <net/neighbour.h> 23 #include <net/arp.h> 24 #include <net/inet_dscp.h> 25 #include <net/ip_fib.h> 26 #include <net/ip6_fib.h> 27 #include <net/nexthop.h> 28 #include <net/fib_rules.h> 29 #include <net/ip_tunnels.h> 30 #include <net/l3mdev.h> 31 #include <net/addrconf.h> 32 #include <net/ndisc.h> 33 #include <net/ipv6.h> 34 #include <net/fib_notifier.h> 35 #include <net/switchdev.h> 36 37 #include "spectrum.h" 38 #include "core.h" 39 #include "reg.h" 40 #include "spectrum_cnt.h" 41 #include "spectrum_dpipe.h" 42 #include "spectrum_ipip.h" 43 #include "spectrum_mr.h" 44 #include "spectrum_mr_tcam.h" 45 #include "spectrum_router.h" 46 #include "spectrum_span.h" 47 48 struct mlxsw_sp_fib; 49 struct mlxsw_sp_vr; 50 struct mlxsw_sp_lpm_tree; 51 struct mlxsw_sp_rif_ops; 52 53 struct mlxsw_sp_rif { 54 struct list_head nexthop_list; 55 struct list_head neigh_list; 56 struct net_device *dev; /* NULL for underlay RIF */ 57 struct mlxsw_sp_fid *fid; 58 unsigned char addr[ETH_ALEN]; 59 int mtu; 60 u16 rif_index; 61 u8 mac_profile_id; 62 u16 vr_id; 63 const struct mlxsw_sp_rif_ops *ops; 64 struct mlxsw_sp *mlxsw_sp; 65 66 unsigned int counter_ingress; 67 bool counter_ingress_valid; 68 unsigned int counter_egress; 69 bool counter_egress_valid; 70 }; 71 72 struct mlxsw_sp_rif_params { 73 struct net_device *dev; 74 union { 75 u16 system_port; 76 u16 lag_id; 77 }; 78 u16 vid; 79 bool lag; 80 }; 81 82 struct mlxsw_sp_rif_subport { 83 struct mlxsw_sp_rif common; 84 refcount_t ref_count; 85 union { 86 u16 system_port; 87 u16 lag_id; 88 }; 89 u16 vid; 90 bool lag; 91 }; 92 93 struct mlxsw_sp_rif_ipip_lb { 94 struct mlxsw_sp_rif common; 95 struct mlxsw_sp_rif_ipip_lb_config lb_config; 96 u16 ul_vr_id; /* Reserved for Spectrum-2. */ 97 u16 ul_rif_id; /* Reserved for Spectrum. */ 98 }; 99 100 struct mlxsw_sp_rif_params_ipip_lb { 101 struct mlxsw_sp_rif_params common; 102 struct mlxsw_sp_rif_ipip_lb_config lb_config; 103 }; 104 105 struct mlxsw_sp_rif_ops { 106 enum mlxsw_sp_rif_type type; 107 size_t rif_size; 108 109 void (*setup)(struct mlxsw_sp_rif *rif, 110 const struct mlxsw_sp_rif_params *params); 111 int (*configure)(struct mlxsw_sp_rif *rif, 112 struct netlink_ext_ack *extack); 113 void (*deconfigure)(struct mlxsw_sp_rif *rif); 114 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif, 115 struct netlink_ext_ack *extack); 116 void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac); 117 }; 118 119 struct mlxsw_sp_rif_mac_profile { 120 unsigned char mac_prefix[ETH_ALEN]; 121 refcount_t ref_count; 122 u8 id; 123 }; 124 125 struct mlxsw_sp_router_ops { 126 int (*init)(struct mlxsw_sp *mlxsw_sp); 127 int (*ipips_init)(struct mlxsw_sp *mlxsw_sp); 128 }; 129 130 static struct mlxsw_sp_rif * 131 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 132 const struct net_device *dev); 133 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif); 134 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree); 135 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 136 struct mlxsw_sp_lpm_tree *lpm_tree); 137 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 138 const struct mlxsw_sp_fib *fib, 139 u8 tree_id); 140 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 141 const struct mlxsw_sp_fib *fib); 142 143 static unsigned int * 144 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif, 145 enum mlxsw_sp_rif_counter_dir dir) 146 { 147 switch (dir) { 148 case MLXSW_SP_RIF_COUNTER_EGRESS: 149 return &rif->counter_egress; 150 case MLXSW_SP_RIF_COUNTER_INGRESS: 151 return &rif->counter_ingress; 152 } 153 return NULL; 154 } 155 156 static bool 157 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif, 158 enum mlxsw_sp_rif_counter_dir dir) 159 { 160 switch (dir) { 161 case MLXSW_SP_RIF_COUNTER_EGRESS: 162 return rif->counter_egress_valid; 163 case MLXSW_SP_RIF_COUNTER_INGRESS: 164 return rif->counter_ingress_valid; 165 } 166 return false; 167 } 168 169 static void 170 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif, 171 enum mlxsw_sp_rif_counter_dir dir, 172 bool valid) 173 { 174 switch (dir) { 175 case MLXSW_SP_RIF_COUNTER_EGRESS: 176 rif->counter_egress_valid = valid; 177 break; 178 case MLXSW_SP_RIF_COUNTER_INGRESS: 179 rif->counter_ingress_valid = valid; 180 break; 181 } 182 } 183 184 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 185 unsigned int counter_index, bool enable, 186 enum mlxsw_sp_rif_counter_dir dir) 187 { 188 char ritr_pl[MLXSW_REG_RITR_LEN]; 189 bool is_egress = false; 190 int err; 191 192 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS) 193 is_egress = true; 194 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 195 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 196 if (err) 197 return err; 198 199 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable, 200 is_egress); 201 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 202 } 203 204 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp, 205 struct mlxsw_sp_rif *rif, 206 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt) 207 { 208 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 209 unsigned int *p_counter_index; 210 bool valid; 211 int err; 212 213 valid = mlxsw_sp_rif_counter_valid_get(rif, dir); 214 if (!valid) 215 return -EINVAL; 216 217 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 218 if (!p_counter_index) 219 return -EINVAL; 220 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 221 MLXSW_REG_RICNT_OPCODE_NOP); 222 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 223 if (err) 224 return err; 225 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl); 226 return 0; 227 } 228 229 struct mlxsw_sp_rif_counter_set_basic { 230 u64 good_unicast_packets; 231 u64 good_multicast_packets; 232 u64 good_broadcast_packets; 233 u64 good_unicast_bytes; 234 u64 good_multicast_bytes; 235 u64 good_broadcast_bytes; 236 u64 error_packets; 237 u64 discard_packets; 238 u64 error_bytes; 239 u64 discard_bytes; 240 }; 241 242 static int 243 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif, 244 enum mlxsw_sp_rif_counter_dir dir, 245 struct mlxsw_sp_rif_counter_set_basic *set) 246 { 247 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 248 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 249 unsigned int *p_counter_index; 250 int err; 251 252 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 253 return -EINVAL; 254 255 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 256 if (!p_counter_index) 257 return -EINVAL; 258 259 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index, 260 MLXSW_REG_RICNT_OPCODE_CLEAR); 261 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 262 if (err) 263 return err; 264 265 if (!set) 266 return 0; 267 268 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME) \ 269 (set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl)) 270 271 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets); 272 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets); 273 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets); 274 MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes); 275 MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes); 276 MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes); 277 MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets); 278 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets); 279 MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes); 280 MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes); 281 282 #undef MLXSW_SP_RIF_COUNTER_EXTRACT 283 284 return 0; 285 } 286 287 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp, 288 unsigned int counter_index) 289 { 290 char ricnt_pl[MLXSW_REG_RICNT_LEN]; 291 292 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index, 293 MLXSW_REG_RICNT_OPCODE_CLEAR); 294 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl); 295 } 296 297 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif, 298 enum mlxsw_sp_rif_counter_dir dir) 299 { 300 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 301 unsigned int *p_counter_index; 302 int err; 303 304 if (mlxsw_sp_rif_counter_valid_get(rif, dir)) 305 return 0; 306 307 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 308 if (!p_counter_index) 309 return -EINVAL; 310 311 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 312 p_counter_index); 313 if (err) 314 return err; 315 316 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index); 317 if (err) 318 goto err_counter_clear; 319 320 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 321 *p_counter_index, true, dir); 322 if (err) 323 goto err_counter_edit; 324 mlxsw_sp_rif_counter_valid_set(rif, dir, true); 325 return 0; 326 327 err_counter_edit: 328 err_counter_clear: 329 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 330 *p_counter_index); 331 return err; 332 } 333 334 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif, 335 enum mlxsw_sp_rif_counter_dir dir) 336 { 337 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 338 unsigned int *p_counter_index; 339 340 if (!mlxsw_sp_rif_counter_valid_get(rif, dir)) 341 return; 342 343 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir); 344 if (WARN_ON(!p_counter_index)) 345 return; 346 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index, 347 *p_counter_index, false, dir); 348 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF, 349 *p_counter_index); 350 mlxsw_sp_rif_counter_valid_set(rif, dir, false); 351 } 352 353 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif) 354 { 355 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 356 struct devlink *devlink; 357 358 devlink = priv_to_devlink(mlxsw_sp->core); 359 if (!devlink_dpipe_table_counter_enabled(devlink, 360 MLXSW_SP_DPIPE_TABLE_NAME_ERIF)) 361 return; 362 mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 363 } 364 365 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif) 366 { 367 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 368 } 369 370 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1) 371 372 struct mlxsw_sp_prefix_usage { 373 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT); 374 }; 375 376 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \ 377 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT) 378 379 static bool 380 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1, 381 struct mlxsw_sp_prefix_usage *prefix_usage2) 382 { 383 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 384 } 385 386 static void 387 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1, 388 struct mlxsw_sp_prefix_usage *prefix_usage2) 389 { 390 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1)); 391 } 392 393 static void 394 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage, 395 unsigned char prefix_len) 396 { 397 set_bit(prefix_len, prefix_usage->b); 398 } 399 400 static void 401 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage, 402 unsigned char prefix_len) 403 { 404 clear_bit(prefix_len, prefix_usage->b); 405 } 406 407 struct mlxsw_sp_fib_key { 408 unsigned char addr[sizeof(struct in6_addr)]; 409 unsigned char prefix_len; 410 }; 411 412 enum mlxsw_sp_fib_entry_type { 413 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE, 414 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL, 415 MLXSW_SP_FIB_ENTRY_TYPE_TRAP, 416 MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE, 417 MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE, 418 419 /* This is a special case of local delivery, where a packet should be 420 * decapsulated on reception. Note that there is no corresponding ENCAP, 421 * because that's a type of next hop, not of FIB entry. (There can be 422 * several next hops in a REMOTE entry, and some of them may be 423 * encapsulating entries.) 424 */ 425 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP, 426 MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP, 427 }; 428 429 struct mlxsw_sp_nexthop_group_info; 430 struct mlxsw_sp_nexthop_group; 431 struct mlxsw_sp_fib_entry; 432 433 struct mlxsw_sp_fib_node { 434 struct mlxsw_sp_fib_entry *fib_entry; 435 struct list_head list; 436 struct rhash_head ht_node; 437 struct mlxsw_sp_fib *fib; 438 struct mlxsw_sp_fib_key key; 439 }; 440 441 struct mlxsw_sp_fib_entry_decap { 442 struct mlxsw_sp_ipip_entry *ipip_entry; 443 u32 tunnel_index; 444 }; 445 446 struct mlxsw_sp_fib_entry { 447 struct mlxsw_sp_fib_node *fib_node; 448 enum mlxsw_sp_fib_entry_type type; 449 struct list_head nexthop_group_node; 450 struct mlxsw_sp_nexthop_group *nh_group; 451 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */ 452 }; 453 454 struct mlxsw_sp_fib4_entry { 455 struct mlxsw_sp_fib_entry common; 456 struct fib_info *fi; 457 u32 tb_id; 458 dscp_t dscp; 459 u8 type; 460 }; 461 462 struct mlxsw_sp_fib6_entry { 463 struct mlxsw_sp_fib_entry common; 464 struct list_head rt6_list; 465 unsigned int nrt6; 466 }; 467 468 struct mlxsw_sp_rt6 { 469 struct list_head list; 470 struct fib6_info *rt; 471 }; 472 473 struct mlxsw_sp_lpm_tree { 474 u8 id; /* tree ID */ 475 unsigned int ref_count; 476 enum mlxsw_sp_l3proto proto; 477 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT]; 478 struct mlxsw_sp_prefix_usage prefix_usage; 479 }; 480 481 struct mlxsw_sp_fib { 482 struct rhashtable ht; 483 struct list_head node_list; 484 struct mlxsw_sp_vr *vr; 485 struct mlxsw_sp_lpm_tree *lpm_tree; 486 enum mlxsw_sp_l3proto proto; 487 }; 488 489 struct mlxsw_sp_vr { 490 u16 id; /* virtual router ID */ 491 u32 tb_id; /* kernel fib table id */ 492 unsigned int rif_count; 493 struct mlxsw_sp_fib *fib4; 494 struct mlxsw_sp_fib *fib6; 495 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX]; 496 struct mlxsw_sp_rif *ul_rif; 497 refcount_t ul_rif_refcnt; 498 }; 499 500 static const struct rhashtable_params mlxsw_sp_fib_ht_params; 501 502 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp, 503 struct mlxsw_sp_vr *vr, 504 enum mlxsw_sp_l3proto proto) 505 { 506 struct mlxsw_sp_lpm_tree *lpm_tree; 507 struct mlxsw_sp_fib *fib; 508 int err; 509 510 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 511 fib = kzalloc(sizeof(*fib), GFP_KERNEL); 512 if (!fib) 513 return ERR_PTR(-ENOMEM); 514 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params); 515 if (err) 516 goto err_rhashtable_init; 517 INIT_LIST_HEAD(&fib->node_list); 518 fib->proto = proto; 519 fib->vr = vr; 520 fib->lpm_tree = lpm_tree; 521 mlxsw_sp_lpm_tree_hold(lpm_tree); 522 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id); 523 if (err) 524 goto err_lpm_tree_bind; 525 return fib; 526 527 err_lpm_tree_bind: 528 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 529 err_rhashtable_init: 530 kfree(fib); 531 return ERR_PTR(err); 532 } 533 534 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp, 535 struct mlxsw_sp_fib *fib) 536 { 537 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib); 538 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree); 539 WARN_ON(!list_empty(&fib->node_list)); 540 rhashtable_destroy(&fib->ht); 541 kfree(fib); 542 } 543 544 static struct mlxsw_sp_lpm_tree * 545 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp) 546 { 547 static struct mlxsw_sp_lpm_tree *lpm_tree; 548 int i; 549 550 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 551 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 552 if (lpm_tree->ref_count == 0) 553 return lpm_tree; 554 } 555 return NULL; 556 } 557 558 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp, 559 struct mlxsw_sp_lpm_tree *lpm_tree) 560 { 561 char ralta_pl[MLXSW_REG_RALTA_LEN]; 562 563 mlxsw_reg_ralta_pack(ralta_pl, true, 564 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 565 lpm_tree->id); 566 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 567 } 568 569 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp, 570 struct mlxsw_sp_lpm_tree *lpm_tree) 571 { 572 char ralta_pl[MLXSW_REG_RALTA_LEN]; 573 574 mlxsw_reg_ralta_pack(ralta_pl, false, 575 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto, 576 lpm_tree->id); 577 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl); 578 } 579 580 static int 581 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp, 582 struct mlxsw_sp_prefix_usage *prefix_usage, 583 struct mlxsw_sp_lpm_tree *lpm_tree) 584 { 585 char ralst_pl[MLXSW_REG_RALST_LEN]; 586 u8 root_bin = 0; 587 u8 prefix; 588 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD; 589 590 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) 591 root_bin = prefix; 592 593 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id); 594 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) { 595 if (prefix == 0) 596 continue; 597 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix, 598 MLXSW_REG_RALST_BIN_NO_CHILD); 599 last_prefix = prefix; 600 } 601 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl); 602 } 603 604 static struct mlxsw_sp_lpm_tree * 605 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, 606 struct mlxsw_sp_prefix_usage *prefix_usage, 607 enum mlxsw_sp_l3proto proto) 608 { 609 struct mlxsw_sp_lpm_tree *lpm_tree; 610 int err; 611 612 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp); 613 if (!lpm_tree) 614 return ERR_PTR(-EBUSY); 615 lpm_tree->proto = proto; 616 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree); 617 if (err) 618 return ERR_PTR(err); 619 620 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage, 621 lpm_tree); 622 if (err) 623 goto err_left_struct_set; 624 memcpy(&lpm_tree->prefix_usage, prefix_usage, 625 sizeof(lpm_tree->prefix_usage)); 626 memset(&lpm_tree->prefix_ref_count, 0, 627 sizeof(lpm_tree->prefix_ref_count)); 628 lpm_tree->ref_count = 1; 629 return lpm_tree; 630 631 err_left_struct_set: 632 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 633 return ERR_PTR(err); 634 } 635 636 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp, 637 struct mlxsw_sp_lpm_tree *lpm_tree) 638 { 639 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree); 640 } 641 642 static struct mlxsw_sp_lpm_tree * 643 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, 644 struct mlxsw_sp_prefix_usage *prefix_usage, 645 enum mlxsw_sp_l3proto proto) 646 { 647 struct mlxsw_sp_lpm_tree *lpm_tree; 648 int i; 649 650 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 651 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 652 if (lpm_tree->ref_count != 0 && 653 lpm_tree->proto == proto && 654 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, 655 prefix_usage)) { 656 mlxsw_sp_lpm_tree_hold(lpm_tree); 657 return lpm_tree; 658 } 659 } 660 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto); 661 } 662 663 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree) 664 { 665 lpm_tree->ref_count++; 666 } 667 668 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp, 669 struct mlxsw_sp_lpm_tree *lpm_tree) 670 { 671 if (--lpm_tree->ref_count == 0) 672 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree); 673 } 674 675 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */ 676 677 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp) 678 { 679 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } }; 680 struct mlxsw_sp_lpm_tree *lpm_tree; 681 u64 max_trees; 682 int err, i; 683 684 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES)) 685 return -EIO; 686 687 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES); 688 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN; 689 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count, 690 sizeof(struct mlxsw_sp_lpm_tree), 691 GFP_KERNEL); 692 if (!mlxsw_sp->router->lpm.trees) 693 return -ENOMEM; 694 695 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) { 696 lpm_tree = &mlxsw_sp->router->lpm.trees[i]; 697 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN; 698 } 699 700 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 701 MLXSW_SP_L3_PROTO_IPV4); 702 if (IS_ERR(lpm_tree)) { 703 err = PTR_ERR(lpm_tree); 704 goto err_ipv4_tree_get; 705 } 706 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree; 707 708 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 709 MLXSW_SP_L3_PROTO_IPV6); 710 if (IS_ERR(lpm_tree)) { 711 err = PTR_ERR(lpm_tree); 712 goto err_ipv6_tree_get; 713 } 714 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree; 715 716 return 0; 717 718 err_ipv6_tree_get: 719 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 720 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 721 err_ipv4_tree_get: 722 kfree(mlxsw_sp->router->lpm.trees); 723 return err; 724 } 725 726 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp) 727 { 728 struct mlxsw_sp_lpm_tree *lpm_tree; 729 730 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6]; 731 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 732 733 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4]; 734 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 735 736 kfree(mlxsw_sp->router->lpm.trees); 737 } 738 739 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr) 740 { 741 return !!vr->fib4 || !!vr->fib6 || 742 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] || 743 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 744 } 745 746 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp) 747 { 748 struct mlxsw_sp_vr *vr; 749 int i; 750 751 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 752 vr = &mlxsw_sp->router->vrs[i]; 753 if (!mlxsw_sp_vr_is_used(vr)) 754 return vr; 755 } 756 return NULL; 757 } 758 759 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp, 760 const struct mlxsw_sp_fib *fib, u8 tree_id) 761 { 762 char raltb_pl[MLXSW_REG_RALTB_LEN]; 763 764 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 765 (enum mlxsw_reg_ralxx_protocol) fib->proto, 766 tree_id); 767 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 768 } 769 770 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp, 771 const struct mlxsw_sp_fib *fib) 772 { 773 char raltb_pl[MLXSW_REG_RALTB_LEN]; 774 775 /* Bind to tree 0 which is default */ 776 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id, 777 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0); 778 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); 779 } 780 781 static u32 mlxsw_sp_fix_tb_id(u32 tb_id) 782 { 783 /* For our purpose, squash main, default and local tables into one */ 784 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT) 785 tb_id = RT_TABLE_MAIN; 786 return tb_id; 787 } 788 789 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp, 790 u32 tb_id) 791 { 792 struct mlxsw_sp_vr *vr; 793 int i; 794 795 tb_id = mlxsw_sp_fix_tb_id(tb_id); 796 797 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 798 vr = &mlxsw_sp->router->vrs[i]; 799 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id) 800 return vr; 801 } 802 return NULL; 803 } 804 805 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 806 u16 *vr_id) 807 { 808 struct mlxsw_sp_vr *vr; 809 int err = 0; 810 811 mutex_lock(&mlxsw_sp->router->lock); 812 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 813 if (!vr) { 814 err = -ESRCH; 815 goto out; 816 } 817 *vr_id = vr->id; 818 out: 819 mutex_unlock(&mlxsw_sp->router->lock); 820 return err; 821 } 822 823 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr, 824 enum mlxsw_sp_l3proto proto) 825 { 826 switch (proto) { 827 case MLXSW_SP_L3_PROTO_IPV4: 828 return vr->fib4; 829 case MLXSW_SP_L3_PROTO_IPV6: 830 return vr->fib6; 831 } 832 return NULL; 833 } 834 835 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp, 836 u32 tb_id, 837 struct netlink_ext_ack *extack) 838 { 839 struct mlxsw_sp_mr_table *mr4_table, *mr6_table; 840 struct mlxsw_sp_fib *fib4; 841 struct mlxsw_sp_fib *fib6; 842 struct mlxsw_sp_vr *vr; 843 int err; 844 845 vr = mlxsw_sp_vr_find_unused(mlxsw_sp); 846 if (!vr) { 847 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers"); 848 return ERR_PTR(-EBUSY); 849 } 850 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 851 if (IS_ERR(fib4)) 852 return ERR_CAST(fib4); 853 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 854 if (IS_ERR(fib6)) { 855 err = PTR_ERR(fib6); 856 goto err_fib6_create; 857 } 858 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 859 MLXSW_SP_L3_PROTO_IPV4); 860 if (IS_ERR(mr4_table)) { 861 err = PTR_ERR(mr4_table); 862 goto err_mr4_table_create; 863 } 864 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id, 865 MLXSW_SP_L3_PROTO_IPV6); 866 if (IS_ERR(mr6_table)) { 867 err = PTR_ERR(mr6_table); 868 goto err_mr6_table_create; 869 } 870 871 vr->fib4 = fib4; 872 vr->fib6 = fib6; 873 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table; 874 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table; 875 vr->tb_id = tb_id; 876 return vr; 877 878 err_mr6_table_create: 879 mlxsw_sp_mr_table_destroy(mr4_table); 880 err_mr4_table_create: 881 mlxsw_sp_fib_destroy(mlxsw_sp, fib6); 882 err_fib6_create: 883 mlxsw_sp_fib_destroy(mlxsw_sp, fib4); 884 return ERR_PTR(err); 885 } 886 887 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp, 888 struct mlxsw_sp_vr *vr) 889 { 890 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]); 891 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL; 892 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]); 893 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL; 894 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6); 895 vr->fib6 = NULL; 896 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4); 897 vr->fib4 = NULL; 898 } 899 900 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 901 struct netlink_ext_ack *extack) 902 { 903 struct mlxsw_sp_vr *vr; 904 905 tb_id = mlxsw_sp_fix_tb_id(tb_id); 906 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 907 if (!vr) 908 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack); 909 return vr; 910 } 911 912 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr) 913 { 914 if (!vr->rif_count && list_empty(&vr->fib4->node_list) && 915 list_empty(&vr->fib6->node_list) && 916 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) && 917 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6])) 918 mlxsw_sp_vr_destroy(mlxsw_sp, vr); 919 } 920 921 static bool 922 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr, 923 enum mlxsw_sp_l3proto proto, u8 tree_id) 924 { 925 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 926 927 if (!mlxsw_sp_vr_is_used(vr)) 928 return false; 929 if (fib->lpm_tree->id == tree_id) 930 return true; 931 return false; 932 } 933 934 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 935 struct mlxsw_sp_fib *fib, 936 struct mlxsw_sp_lpm_tree *new_tree) 937 { 938 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree; 939 int err; 940 941 fib->lpm_tree = new_tree; 942 mlxsw_sp_lpm_tree_hold(new_tree); 943 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id); 944 if (err) 945 goto err_tree_bind; 946 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 947 return 0; 948 949 err_tree_bind: 950 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree); 951 fib->lpm_tree = old_tree; 952 return err; 953 } 954 955 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp, 956 struct mlxsw_sp_fib *fib, 957 struct mlxsw_sp_lpm_tree *new_tree) 958 { 959 enum mlxsw_sp_l3proto proto = fib->proto; 960 struct mlxsw_sp_lpm_tree *old_tree; 961 u8 old_id, new_id = new_tree->id; 962 struct mlxsw_sp_vr *vr; 963 int i, err; 964 965 old_tree = mlxsw_sp->router->lpm.proto_trees[proto]; 966 old_id = old_tree->id; 967 968 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 969 vr = &mlxsw_sp->router->vrs[i]; 970 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id)) 971 continue; 972 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 973 mlxsw_sp_vr_fib(vr, proto), 974 new_tree); 975 if (err) 976 goto err_tree_replace; 977 } 978 979 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count, 980 sizeof(new_tree->prefix_ref_count)); 981 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree; 982 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree); 983 984 return 0; 985 986 err_tree_replace: 987 for (i--; i >= 0; i--) { 988 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id)) 989 continue; 990 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp, 991 mlxsw_sp_vr_fib(vr, proto), 992 old_tree); 993 } 994 return err; 995 } 996 997 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp) 998 { 999 struct mlxsw_sp_vr *vr; 1000 u64 max_vrs; 1001 int i; 1002 1003 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS)) 1004 return -EIO; 1005 1006 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); 1007 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr), 1008 GFP_KERNEL); 1009 if (!mlxsw_sp->router->vrs) 1010 return -ENOMEM; 1011 1012 for (i = 0; i < max_vrs; i++) { 1013 vr = &mlxsw_sp->router->vrs[i]; 1014 vr->id = i; 1015 } 1016 1017 return 0; 1018 } 1019 1020 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp); 1021 1022 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp) 1023 { 1024 /* At this stage we're guaranteed not to have new incoming 1025 * FIB notifications and the work queue is free from FIBs 1026 * sitting on top of mlxsw netdevs. However, we can still 1027 * have other FIBs queued. Flush the queue before flushing 1028 * the device's tables. No need for locks, as we're the only 1029 * writer. 1030 */ 1031 mlxsw_core_flush_owq(); 1032 mlxsw_sp_router_fib_flush(mlxsw_sp); 1033 kfree(mlxsw_sp->router->vrs); 1034 } 1035 1036 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev) 1037 { 1038 struct net_device *d; 1039 u32 tb_id; 1040 1041 rcu_read_lock(); 1042 d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1043 if (d) 1044 tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN; 1045 else 1046 tb_id = RT_TABLE_MAIN; 1047 rcu_read_unlock(); 1048 1049 return tb_id; 1050 } 1051 1052 static struct mlxsw_sp_rif * 1053 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 1054 const struct mlxsw_sp_rif_params *params, 1055 struct netlink_ext_ack *extack); 1056 1057 static struct mlxsw_sp_rif_ipip_lb * 1058 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp, 1059 enum mlxsw_sp_ipip_type ipipt, 1060 struct net_device *ol_dev, 1061 struct netlink_ext_ack *extack) 1062 { 1063 struct mlxsw_sp_rif_params_ipip_lb lb_params; 1064 const struct mlxsw_sp_ipip_ops *ipip_ops; 1065 struct mlxsw_sp_rif *rif; 1066 1067 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1068 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) { 1069 .common.dev = ol_dev, 1070 .common.lag = false, 1071 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev), 1072 }; 1073 1074 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack); 1075 if (IS_ERR(rif)) 1076 return ERR_CAST(rif); 1077 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 1078 } 1079 1080 static struct mlxsw_sp_ipip_entry * 1081 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp, 1082 enum mlxsw_sp_ipip_type ipipt, 1083 struct net_device *ol_dev) 1084 { 1085 const struct mlxsw_sp_ipip_ops *ipip_ops; 1086 struct mlxsw_sp_ipip_entry *ipip_entry; 1087 struct mlxsw_sp_ipip_entry *ret = NULL; 1088 int err; 1089 1090 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1091 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL); 1092 if (!ipip_entry) 1093 return ERR_PTR(-ENOMEM); 1094 1095 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt, 1096 ol_dev, NULL); 1097 if (IS_ERR(ipip_entry->ol_lb)) { 1098 ret = ERR_CAST(ipip_entry->ol_lb); 1099 goto err_ol_ipip_lb_create; 1100 } 1101 1102 ipip_entry->ipipt = ipipt; 1103 ipip_entry->ol_dev = ol_dev; 1104 ipip_entry->parms = ipip_ops->parms_init(ol_dev); 1105 1106 err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry); 1107 if (err) { 1108 ret = ERR_PTR(err); 1109 goto err_rem_ip_addr_set; 1110 } 1111 1112 return ipip_entry; 1113 1114 err_rem_ip_addr_set: 1115 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1116 err_ol_ipip_lb_create: 1117 kfree(ipip_entry); 1118 return ret; 1119 } 1120 1121 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp, 1122 struct mlxsw_sp_ipip_entry *ipip_entry) 1123 { 1124 const struct mlxsw_sp_ipip_ops *ipip_ops = 1125 mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1126 1127 ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry); 1128 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common); 1129 kfree(ipip_entry); 1130 } 1131 1132 static bool 1133 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp, 1134 const enum mlxsw_sp_l3proto ul_proto, 1135 union mlxsw_sp_l3addr saddr, 1136 u32 ul_tb_id, 1137 struct mlxsw_sp_ipip_entry *ipip_entry) 1138 { 1139 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1140 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1141 union mlxsw_sp_l3addr tun_saddr; 1142 1143 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1144 return false; 1145 1146 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1147 return tun_ul_tb_id == ul_tb_id && 1148 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr); 1149 } 1150 1151 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp, 1152 enum mlxsw_sp_ipip_type ipipt) 1153 { 1154 const struct mlxsw_sp_ipip_ops *ipip_ops; 1155 1156 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1157 1158 /* Not all tunnels require to increase the default pasing depth 1159 * (96 bytes). 1160 */ 1161 if (ipip_ops->inc_parsing_depth) 1162 return mlxsw_sp_parsing_depth_inc(mlxsw_sp); 1163 1164 return 0; 1165 } 1166 1167 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp, 1168 enum mlxsw_sp_ipip_type ipipt) 1169 { 1170 const struct mlxsw_sp_ipip_ops *ipip_ops = 1171 mlxsw_sp->router->ipip_ops_arr[ipipt]; 1172 1173 if (ipip_ops->inc_parsing_depth) 1174 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 1175 } 1176 1177 static int 1178 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp, 1179 struct mlxsw_sp_fib_entry *fib_entry, 1180 struct mlxsw_sp_ipip_entry *ipip_entry) 1181 { 1182 u32 tunnel_index; 1183 int err; 1184 1185 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1186 1, &tunnel_index); 1187 if (err) 1188 return err; 1189 1190 err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp, 1191 ipip_entry->ipipt); 1192 if (err) 1193 goto err_parsing_depth_inc; 1194 1195 ipip_entry->decap_fib_entry = fib_entry; 1196 fib_entry->decap.ipip_entry = ipip_entry; 1197 fib_entry->decap.tunnel_index = tunnel_index; 1198 1199 return 0; 1200 1201 err_parsing_depth_inc: 1202 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 1203 fib_entry->decap.tunnel_index); 1204 return err; 1205 } 1206 1207 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp, 1208 struct mlxsw_sp_fib_entry *fib_entry) 1209 { 1210 enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt; 1211 1212 /* Unlink this node from the IPIP entry that it's the decap entry of. */ 1213 fib_entry->decap.ipip_entry->decap_fib_entry = NULL; 1214 fib_entry->decap.ipip_entry = NULL; 1215 mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt); 1216 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1217 1, fib_entry->decap.tunnel_index); 1218 } 1219 1220 static struct mlxsw_sp_fib_node * 1221 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 1222 size_t addr_len, unsigned char prefix_len); 1223 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 1224 struct mlxsw_sp_fib_entry *fib_entry); 1225 1226 static void 1227 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp, 1228 struct mlxsw_sp_ipip_entry *ipip_entry) 1229 { 1230 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry; 1231 1232 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 1233 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1234 1235 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 1236 } 1237 1238 static void 1239 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp, 1240 struct mlxsw_sp_ipip_entry *ipip_entry, 1241 struct mlxsw_sp_fib_entry *decap_fib_entry) 1242 { 1243 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry, 1244 ipip_entry)) 1245 return; 1246 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 1247 1248 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry)) 1249 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1250 } 1251 1252 static struct mlxsw_sp_fib_entry * 1253 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 1254 enum mlxsw_sp_l3proto proto, 1255 const union mlxsw_sp_l3addr *addr, 1256 enum mlxsw_sp_fib_entry_type type) 1257 { 1258 struct mlxsw_sp_fib_node *fib_node; 1259 unsigned char addr_prefix_len; 1260 struct mlxsw_sp_fib *fib; 1261 struct mlxsw_sp_vr *vr; 1262 const void *addrp; 1263 size_t addr_len; 1264 u32 addr4; 1265 1266 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id); 1267 if (!vr) 1268 return NULL; 1269 fib = mlxsw_sp_vr_fib(vr, proto); 1270 1271 switch (proto) { 1272 case MLXSW_SP_L3_PROTO_IPV4: 1273 addr4 = be32_to_cpu(addr->addr4); 1274 addrp = &addr4; 1275 addr_len = 4; 1276 addr_prefix_len = 32; 1277 break; 1278 case MLXSW_SP_L3_PROTO_IPV6: 1279 addrp = &addr->addr6; 1280 addr_len = 16; 1281 addr_prefix_len = 128; 1282 break; 1283 default: 1284 WARN_ON(1); 1285 return NULL; 1286 } 1287 1288 fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len, 1289 addr_prefix_len); 1290 if (!fib_node || fib_node->fib_entry->type != type) 1291 return NULL; 1292 1293 return fib_node->fib_entry; 1294 } 1295 1296 /* Given an IPIP entry, find the corresponding decap route. */ 1297 static struct mlxsw_sp_fib_entry * 1298 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp, 1299 struct mlxsw_sp_ipip_entry *ipip_entry) 1300 { 1301 static struct mlxsw_sp_fib_node *fib_node; 1302 const struct mlxsw_sp_ipip_ops *ipip_ops; 1303 unsigned char saddr_prefix_len; 1304 union mlxsw_sp_l3addr saddr; 1305 struct mlxsw_sp_fib *ul_fib; 1306 struct mlxsw_sp_vr *ul_vr; 1307 const void *saddrp; 1308 size_t saddr_len; 1309 u32 ul_tb_id; 1310 u32 saddr4; 1311 1312 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1313 1314 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev); 1315 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id); 1316 if (!ul_vr) 1317 return NULL; 1318 1319 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto); 1320 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto, 1321 ipip_entry->ol_dev); 1322 1323 switch (ipip_ops->ul_proto) { 1324 case MLXSW_SP_L3_PROTO_IPV4: 1325 saddr4 = be32_to_cpu(saddr.addr4); 1326 saddrp = &saddr4; 1327 saddr_len = 4; 1328 saddr_prefix_len = 32; 1329 break; 1330 case MLXSW_SP_L3_PROTO_IPV6: 1331 saddrp = &saddr.addr6; 1332 saddr_len = 16; 1333 saddr_prefix_len = 128; 1334 break; 1335 default: 1336 WARN_ON(1); 1337 return NULL; 1338 } 1339 1340 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len, 1341 saddr_prefix_len); 1342 if (!fib_node || 1343 fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP) 1344 return NULL; 1345 1346 return fib_node->fib_entry; 1347 } 1348 1349 static struct mlxsw_sp_ipip_entry * 1350 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp, 1351 enum mlxsw_sp_ipip_type ipipt, 1352 struct net_device *ol_dev) 1353 { 1354 struct mlxsw_sp_ipip_entry *ipip_entry; 1355 1356 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev); 1357 if (IS_ERR(ipip_entry)) 1358 return ipip_entry; 1359 1360 list_add_tail(&ipip_entry->ipip_list_node, 1361 &mlxsw_sp->router->ipip_list); 1362 1363 return ipip_entry; 1364 } 1365 1366 static void 1367 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp, 1368 struct mlxsw_sp_ipip_entry *ipip_entry) 1369 { 1370 list_del(&ipip_entry->ipip_list_node); 1371 mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry); 1372 } 1373 1374 static bool 1375 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp, 1376 const struct net_device *ul_dev, 1377 enum mlxsw_sp_l3proto ul_proto, 1378 union mlxsw_sp_l3addr ul_dip, 1379 struct mlxsw_sp_ipip_entry *ipip_entry) 1380 { 1381 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1382 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt; 1383 1384 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto) 1385 return false; 1386 1387 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip, 1388 ul_tb_id, ipip_entry); 1389 } 1390 1391 /* Given decap parameters, find the corresponding IPIP entry. */ 1392 static struct mlxsw_sp_ipip_entry * 1393 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex, 1394 enum mlxsw_sp_l3proto ul_proto, 1395 union mlxsw_sp_l3addr ul_dip) 1396 { 1397 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1398 struct net_device *ul_dev; 1399 1400 rcu_read_lock(); 1401 1402 ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex); 1403 if (!ul_dev) 1404 goto out_unlock; 1405 1406 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1407 ipip_list_node) 1408 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev, 1409 ul_proto, ul_dip, 1410 ipip_entry)) 1411 goto out_unlock; 1412 1413 rcu_read_unlock(); 1414 1415 return NULL; 1416 1417 out_unlock: 1418 rcu_read_unlock(); 1419 return ipip_entry; 1420 } 1421 1422 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp, 1423 const struct net_device *dev, 1424 enum mlxsw_sp_ipip_type *p_type) 1425 { 1426 struct mlxsw_sp_router *router = mlxsw_sp->router; 1427 const struct mlxsw_sp_ipip_ops *ipip_ops; 1428 enum mlxsw_sp_ipip_type ipipt; 1429 1430 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) { 1431 ipip_ops = router->ipip_ops_arr[ipipt]; 1432 if (dev->type == ipip_ops->dev_type) { 1433 if (p_type) 1434 *p_type = ipipt; 1435 return true; 1436 } 1437 } 1438 return false; 1439 } 1440 1441 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp, 1442 const struct net_device *dev) 1443 { 1444 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 1445 } 1446 1447 static struct mlxsw_sp_ipip_entry * 1448 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp, 1449 const struct net_device *ol_dev) 1450 { 1451 struct mlxsw_sp_ipip_entry *ipip_entry; 1452 1453 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list, 1454 ipip_list_node) 1455 if (ipip_entry->ol_dev == ol_dev) 1456 return ipip_entry; 1457 1458 return NULL; 1459 } 1460 1461 static struct mlxsw_sp_ipip_entry * 1462 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp, 1463 const struct net_device *ul_dev, 1464 struct mlxsw_sp_ipip_entry *start) 1465 { 1466 struct mlxsw_sp_ipip_entry *ipip_entry; 1467 1468 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list, 1469 ipip_list_node); 1470 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list, 1471 ipip_list_node) { 1472 struct net_device *ol_dev = ipip_entry->ol_dev; 1473 struct net_device *ipip_ul_dev; 1474 1475 rcu_read_lock(); 1476 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1477 rcu_read_unlock(); 1478 1479 if (ipip_ul_dev == ul_dev) 1480 return ipip_entry; 1481 } 1482 1483 return NULL; 1484 } 1485 1486 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp, 1487 const struct net_device *dev) 1488 { 1489 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL); 1490 } 1491 1492 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp, 1493 const struct net_device *ol_dev, 1494 enum mlxsw_sp_ipip_type ipipt) 1495 { 1496 const struct mlxsw_sp_ipip_ops *ops 1497 = mlxsw_sp->router->ipip_ops_arr[ipipt]; 1498 1499 return ops->can_offload(mlxsw_sp, ol_dev); 1500 } 1501 1502 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp, 1503 struct net_device *ol_dev) 1504 { 1505 enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX; 1506 struct mlxsw_sp_ipip_entry *ipip_entry; 1507 enum mlxsw_sp_l3proto ul_proto; 1508 union mlxsw_sp_l3addr saddr; 1509 u32 ul_tb_id; 1510 1511 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt); 1512 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) { 1513 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev); 1514 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto; 1515 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev); 1516 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1517 saddr, ul_tb_id, 1518 NULL)) { 1519 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt, 1520 ol_dev); 1521 if (IS_ERR(ipip_entry)) 1522 return PTR_ERR(ipip_entry); 1523 } 1524 } 1525 1526 return 0; 1527 } 1528 1529 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp, 1530 struct net_device *ol_dev) 1531 { 1532 struct mlxsw_sp_ipip_entry *ipip_entry; 1533 1534 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1535 if (ipip_entry) 1536 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1537 } 1538 1539 static void 1540 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1541 struct mlxsw_sp_ipip_entry *ipip_entry) 1542 { 1543 struct mlxsw_sp_fib_entry *decap_fib_entry; 1544 1545 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry); 1546 if (decap_fib_entry) 1547 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry, 1548 decap_fib_entry); 1549 } 1550 1551 static int 1552 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id, 1553 u16 ul_rif_id, bool enable) 1554 { 1555 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config; 1556 enum mlxsw_reg_ritr_loopback_ipip_options ipip_options; 1557 struct mlxsw_sp_rif *rif = &lb_rif->common; 1558 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 1559 char ritr_pl[MLXSW_REG_RITR_LEN]; 1560 struct in6_addr *saddr6; 1561 u32 saddr4; 1562 1563 ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET; 1564 switch (lb_cf.ul_protocol) { 1565 case MLXSW_SP_L3_PROTO_IPV4: 1566 saddr4 = be32_to_cpu(lb_cf.saddr.addr4); 1567 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1568 rif->rif_index, rif->vr_id, rif->dev->mtu); 1569 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt, 1570 ipip_options, ul_vr_id, 1571 ul_rif_id, saddr4, 1572 lb_cf.okey); 1573 break; 1574 1575 case MLXSW_SP_L3_PROTO_IPV6: 1576 saddr6 = &lb_cf.saddr.addr6; 1577 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 1578 rif->rif_index, rif->vr_id, rif->dev->mtu); 1579 mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt, 1580 ipip_options, ul_vr_id, 1581 ul_rif_id, saddr6, 1582 lb_cf.okey); 1583 break; 1584 } 1585 1586 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 1587 } 1588 1589 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp, 1590 struct net_device *ol_dev) 1591 { 1592 struct mlxsw_sp_ipip_entry *ipip_entry; 1593 struct mlxsw_sp_rif_ipip_lb *lb_rif; 1594 int err = 0; 1595 1596 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1597 if (ipip_entry) { 1598 lb_rif = ipip_entry->ol_lb; 1599 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id, 1600 lb_rif->ul_rif_id, true); 1601 if (err) 1602 goto out; 1603 lb_rif->common.mtu = ol_dev->mtu; 1604 } 1605 1606 out: 1607 return err; 1608 } 1609 1610 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp, 1611 struct net_device *ol_dev) 1612 { 1613 struct mlxsw_sp_ipip_entry *ipip_entry; 1614 1615 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1616 if (ipip_entry) 1617 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1618 } 1619 1620 static void 1621 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1622 struct mlxsw_sp_ipip_entry *ipip_entry) 1623 { 1624 if (ipip_entry->decap_fib_entry) 1625 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1626 } 1627 1628 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp, 1629 struct net_device *ol_dev) 1630 { 1631 struct mlxsw_sp_ipip_entry *ipip_entry; 1632 1633 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1634 if (ipip_entry) 1635 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1636 } 1637 1638 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 1639 struct mlxsw_sp_rif *old_rif, 1640 struct mlxsw_sp_rif *new_rif); 1641 static int 1642 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp, 1643 struct mlxsw_sp_ipip_entry *ipip_entry, 1644 bool keep_encap, 1645 struct netlink_ext_ack *extack) 1646 { 1647 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb; 1648 struct mlxsw_sp_rif_ipip_lb *new_lb_rif; 1649 1650 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, 1651 ipip_entry->ipipt, 1652 ipip_entry->ol_dev, 1653 extack); 1654 if (IS_ERR(new_lb_rif)) 1655 return PTR_ERR(new_lb_rif); 1656 ipip_entry->ol_lb = new_lb_rif; 1657 1658 if (keep_encap) 1659 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common, 1660 &new_lb_rif->common); 1661 1662 mlxsw_sp_rif_destroy(&old_lb_rif->common); 1663 1664 return 0; 1665 } 1666 1667 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 1668 struct mlxsw_sp_rif *rif); 1669 1670 /** 1671 * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry. 1672 * @mlxsw_sp: mlxsw_sp. 1673 * @ipip_entry: IPIP entry. 1674 * @recreate_loopback: Recreates the associated loopback RIF. 1675 * @keep_encap: Updates next hops that use the tunnel netdevice. This is only 1676 * relevant when recreate_loopback is true. 1677 * @update_nexthops: Updates next hops, keeping the current loopback RIF. This 1678 * is only relevant when recreate_loopback is false. 1679 * @extack: extack. 1680 * 1681 * Return: Non-zero value on failure. 1682 */ 1683 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp, 1684 struct mlxsw_sp_ipip_entry *ipip_entry, 1685 bool recreate_loopback, 1686 bool keep_encap, 1687 bool update_nexthops, 1688 struct netlink_ext_ack *extack) 1689 { 1690 int err; 1691 1692 /* RIFs can't be edited, so to update loopback, we need to destroy and 1693 * recreate it. That creates a window of opportunity where RALUE and 1694 * RATR registers end up referencing a RIF that's already gone. RATRs 1695 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care 1696 * of RALUE, demote the decap route back. 1697 */ 1698 if (ipip_entry->decap_fib_entry) 1699 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry); 1700 1701 if (recreate_loopback) { 1702 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry, 1703 keep_encap, extack); 1704 if (err) 1705 return err; 1706 } else if (update_nexthops) { 1707 mlxsw_sp_nexthop_rif_update(mlxsw_sp, 1708 &ipip_entry->ol_lb->common); 1709 } 1710 1711 if (ipip_entry->ol_dev->flags & IFF_UP) 1712 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry); 1713 1714 return 0; 1715 } 1716 1717 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp, 1718 struct net_device *ol_dev, 1719 struct netlink_ext_ack *extack) 1720 { 1721 struct mlxsw_sp_ipip_entry *ipip_entry = 1722 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1723 1724 if (!ipip_entry) 1725 return 0; 1726 1727 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1728 true, false, false, extack); 1729 } 1730 1731 static int 1732 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp, 1733 struct mlxsw_sp_ipip_entry *ipip_entry, 1734 struct net_device *ul_dev, 1735 bool *demote_this, 1736 struct netlink_ext_ack *extack) 1737 { 1738 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN; 1739 enum mlxsw_sp_l3proto ul_proto; 1740 union mlxsw_sp_l3addr saddr; 1741 1742 /* Moving underlay to a different VRF might cause local address 1743 * conflict, and the conflicting tunnels need to be demoted. 1744 */ 1745 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto; 1746 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev); 1747 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto, 1748 saddr, ul_tb_id, 1749 ipip_entry)) { 1750 *demote_this = true; 1751 return 0; 1752 } 1753 1754 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1755 true, true, false, extack); 1756 } 1757 1758 static int 1759 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp, 1760 struct mlxsw_sp_ipip_entry *ipip_entry, 1761 struct net_device *ul_dev) 1762 { 1763 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1764 false, false, true, NULL); 1765 } 1766 1767 static int 1768 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp, 1769 struct mlxsw_sp_ipip_entry *ipip_entry, 1770 struct net_device *ul_dev) 1771 { 1772 /* A down underlay device causes encapsulated packets to not be 1773 * forwarded, but decap still works. So refresh next hops without 1774 * touching anything else. 1775 */ 1776 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry, 1777 false, false, true, NULL); 1778 } 1779 1780 static int 1781 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp, 1782 struct net_device *ol_dev, 1783 struct netlink_ext_ack *extack) 1784 { 1785 const struct mlxsw_sp_ipip_ops *ipip_ops; 1786 struct mlxsw_sp_ipip_entry *ipip_entry; 1787 int err; 1788 1789 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev); 1790 if (!ipip_entry) 1791 /* A change might make a tunnel eligible for offloading, but 1792 * that is currently not implemented. What falls to slow path 1793 * stays there. 1794 */ 1795 return 0; 1796 1797 /* A change might make a tunnel not eligible for offloading. */ 1798 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, 1799 ipip_entry->ipipt)) { 1800 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1801 return 0; 1802 } 1803 1804 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 1805 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack); 1806 return err; 1807 } 1808 1809 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp, 1810 struct mlxsw_sp_ipip_entry *ipip_entry) 1811 { 1812 struct net_device *ol_dev = ipip_entry->ol_dev; 1813 1814 if (ol_dev->flags & IFF_UP) 1815 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry); 1816 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry); 1817 } 1818 1819 /* The configuration where several tunnels have the same local address in the 1820 * same underlay table needs special treatment in the HW. That is currently not 1821 * implemented in the driver. This function finds and demotes the first tunnel 1822 * with a given source address, except the one passed in in the argument 1823 * `except'. 1824 */ 1825 bool 1826 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp, 1827 enum mlxsw_sp_l3proto ul_proto, 1828 union mlxsw_sp_l3addr saddr, 1829 u32 ul_tb_id, 1830 const struct mlxsw_sp_ipip_entry *except) 1831 { 1832 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1833 1834 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1835 ipip_list_node) { 1836 if (ipip_entry != except && 1837 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr, 1838 ul_tb_id, ipip_entry)) { 1839 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1840 return true; 1841 } 1842 } 1843 1844 return false; 1845 } 1846 1847 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp, 1848 struct net_device *ul_dev) 1849 { 1850 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp; 1851 1852 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list, 1853 ipip_list_node) { 1854 struct net_device *ol_dev = ipip_entry->ol_dev; 1855 struct net_device *ipip_ul_dev; 1856 1857 rcu_read_lock(); 1858 ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 1859 rcu_read_unlock(); 1860 if (ipip_ul_dev == ul_dev) 1861 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1862 } 1863 } 1864 1865 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp, 1866 struct net_device *ol_dev, 1867 unsigned long event, 1868 struct netdev_notifier_info *info) 1869 { 1870 struct netdev_notifier_changeupper_info *chup; 1871 struct netlink_ext_ack *extack; 1872 int err = 0; 1873 1874 switch (event) { 1875 case NETDEV_REGISTER: 1876 err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev); 1877 break; 1878 case NETDEV_UNREGISTER: 1879 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev); 1880 break; 1881 case NETDEV_UP: 1882 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev); 1883 break; 1884 case NETDEV_DOWN: 1885 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev); 1886 break; 1887 case NETDEV_CHANGEUPPER: 1888 chup = container_of(info, typeof(*chup), info); 1889 extack = info->extack; 1890 if (netif_is_l3_master(chup->upper_dev)) 1891 err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp, 1892 ol_dev, 1893 extack); 1894 break; 1895 case NETDEV_CHANGE: 1896 extack = info->extack; 1897 err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp, 1898 ol_dev, extack); 1899 break; 1900 case NETDEV_CHANGEMTU: 1901 err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev); 1902 break; 1903 } 1904 return err; 1905 } 1906 1907 static int 1908 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1909 struct mlxsw_sp_ipip_entry *ipip_entry, 1910 struct net_device *ul_dev, 1911 bool *demote_this, 1912 unsigned long event, 1913 struct netdev_notifier_info *info) 1914 { 1915 struct netdev_notifier_changeupper_info *chup; 1916 struct netlink_ext_ack *extack; 1917 1918 switch (event) { 1919 case NETDEV_CHANGEUPPER: 1920 chup = container_of(info, typeof(*chup), info); 1921 extack = info->extack; 1922 if (netif_is_l3_master(chup->upper_dev)) 1923 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp, 1924 ipip_entry, 1925 ul_dev, 1926 demote_this, 1927 extack); 1928 break; 1929 1930 case NETDEV_UP: 1931 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry, 1932 ul_dev); 1933 case NETDEV_DOWN: 1934 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp, 1935 ipip_entry, 1936 ul_dev); 1937 } 1938 return 0; 1939 } 1940 1941 static int 1942 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp, 1943 struct net_device *ul_dev, 1944 unsigned long event, 1945 struct netdev_notifier_info *info) 1946 { 1947 struct mlxsw_sp_ipip_entry *ipip_entry = NULL; 1948 int err; 1949 1950 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, 1951 ul_dev, 1952 ipip_entry))) { 1953 struct mlxsw_sp_ipip_entry *prev; 1954 bool demote_this = false; 1955 1956 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry, 1957 ul_dev, &demote_this, 1958 event, info); 1959 if (err) { 1960 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp, 1961 ul_dev); 1962 return err; 1963 } 1964 1965 if (demote_this) { 1966 if (list_is_first(&ipip_entry->ipip_list_node, 1967 &mlxsw_sp->router->ipip_list)) 1968 prev = NULL; 1969 else 1970 /* This can't be cached from previous iteration, 1971 * because that entry could be gone now. 1972 */ 1973 prev = list_prev_entry(ipip_entry, 1974 ipip_list_node); 1975 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry); 1976 ipip_entry = prev; 1977 } 1978 } 1979 1980 return 0; 1981 } 1982 1983 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 1984 enum mlxsw_sp_l3proto ul_proto, 1985 const union mlxsw_sp_l3addr *ul_sip, 1986 u32 tunnel_index) 1987 { 1988 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 1989 struct mlxsw_sp_router *router = mlxsw_sp->router; 1990 struct mlxsw_sp_fib_entry *fib_entry; 1991 int err = 0; 1992 1993 mutex_lock(&mlxsw_sp->router->lock); 1994 1995 if (WARN_ON_ONCE(router->nve_decap_config.valid)) { 1996 err = -EINVAL; 1997 goto out; 1998 } 1999 2000 router->nve_decap_config.ul_tb_id = ul_tb_id; 2001 router->nve_decap_config.tunnel_index = tunnel_index; 2002 router->nve_decap_config.ul_proto = ul_proto; 2003 router->nve_decap_config.ul_sip = *ul_sip; 2004 router->nve_decap_config.valid = true; 2005 2006 /* It is valid to create a tunnel with a local IP and only later 2007 * assign this IP address to a local interface 2008 */ 2009 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2010 ul_proto, ul_sip, 2011 type); 2012 if (!fib_entry) 2013 goto out; 2014 2015 fib_entry->decap.tunnel_index = tunnel_index; 2016 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2017 2018 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2019 if (err) 2020 goto err_fib_entry_update; 2021 2022 goto out; 2023 2024 err_fib_entry_update: 2025 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2026 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2027 out: 2028 mutex_unlock(&mlxsw_sp->router->lock); 2029 return err; 2030 } 2031 2032 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 2033 enum mlxsw_sp_l3proto ul_proto, 2034 const union mlxsw_sp_l3addr *ul_sip) 2035 { 2036 enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 2037 struct mlxsw_sp_router *router = mlxsw_sp->router; 2038 struct mlxsw_sp_fib_entry *fib_entry; 2039 2040 mutex_lock(&mlxsw_sp->router->lock); 2041 2042 if (WARN_ON_ONCE(!router->nve_decap_config.valid)) 2043 goto out; 2044 2045 router->nve_decap_config.valid = false; 2046 2047 fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id, 2048 ul_proto, ul_sip, 2049 type); 2050 if (!fib_entry) 2051 goto out; 2052 2053 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 2054 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 2055 out: 2056 mutex_unlock(&mlxsw_sp->router->lock); 2057 } 2058 2059 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp, 2060 u32 ul_tb_id, 2061 enum mlxsw_sp_l3proto ul_proto, 2062 const union mlxsw_sp_l3addr *ul_sip) 2063 { 2064 struct mlxsw_sp_router *router = mlxsw_sp->router; 2065 2066 return router->nve_decap_config.valid && 2067 router->nve_decap_config.ul_tb_id == ul_tb_id && 2068 router->nve_decap_config.ul_proto == ul_proto && 2069 !memcmp(&router->nve_decap_config.ul_sip, ul_sip, 2070 sizeof(*ul_sip)); 2071 } 2072 2073 struct mlxsw_sp_neigh_key { 2074 struct neighbour *n; 2075 }; 2076 2077 struct mlxsw_sp_neigh_entry { 2078 struct list_head rif_list_node; 2079 struct rhash_head ht_node; 2080 struct mlxsw_sp_neigh_key key; 2081 u16 rif; 2082 bool connected; 2083 unsigned char ha[ETH_ALEN]; 2084 struct list_head nexthop_list; /* list of nexthops using 2085 * this neigh entry 2086 */ 2087 struct list_head nexthop_neighs_list_node; 2088 unsigned int counter_index; 2089 bool counter_valid; 2090 }; 2091 2092 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = { 2093 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key), 2094 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node), 2095 .key_len = sizeof(struct mlxsw_sp_neigh_key), 2096 }; 2097 2098 struct mlxsw_sp_neigh_entry * 2099 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif, 2100 struct mlxsw_sp_neigh_entry *neigh_entry) 2101 { 2102 if (!neigh_entry) { 2103 if (list_empty(&rif->neigh_list)) 2104 return NULL; 2105 else 2106 return list_first_entry(&rif->neigh_list, 2107 typeof(*neigh_entry), 2108 rif_list_node); 2109 } 2110 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list)) 2111 return NULL; 2112 return list_next_entry(neigh_entry, rif_list_node); 2113 } 2114 2115 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry) 2116 { 2117 return neigh_entry->key.n->tbl->family; 2118 } 2119 2120 unsigned char * 2121 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry) 2122 { 2123 return neigh_entry->ha; 2124 } 2125 2126 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2127 { 2128 struct neighbour *n; 2129 2130 n = neigh_entry->key.n; 2131 return ntohl(*((__be32 *) n->primary_key)); 2132 } 2133 2134 struct in6_addr * 2135 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry) 2136 { 2137 struct neighbour *n; 2138 2139 n = neigh_entry->key.n; 2140 return (struct in6_addr *) &n->primary_key; 2141 } 2142 2143 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp, 2144 struct mlxsw_sp_neigh_entry *neigh_entry, 2145 u64 *p_counter) 2146 { 2147 if (!neigh_entry->counter_valid) 2148 return -EINVAL; 2149 2150 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index, 2151 p_counter, NULL); 2152 } 2153 2154 static struct mlxsw_sp_neigh_entry * 2155 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n, 2156 u16 rif) 2157 { 2158 struct mlxsw_sp_neigh_entry *neigh_entry; 2159 2160 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL); 2161 if (!neigh_entry) 2162 return NULL; 2163 2164 neigh_entry->key.n = n; 2165 neigh_entry->rif = rif; 2166 INIT_LIST_HEAD(&neigh_entry->nexthop_list); 2167 2168 return neigh_entry; 2169 } 2170 2171 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry) 2172 { 2173 kfree(neigh_entry); 2174 } 2175 2176 static int 2177 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp, 2178 struct mlxsw_sp_neigh_entry *neigh_entry) 2179 { 2180 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht, 2181 &neigh_entry->ht_node, 2182 mlxsw_sp_neigh_ht_params); 2183 } 2184 2185 static void 2186 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp, 2187 struct mlxsw_sp_neigh_entry *neigh_entry) 2188 { 2189 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht, 2190 &neigh_entry->ht_node, 2191 mlxsw_sp_neigh_ht_params); 2192 } 2193 2194 static bool 2195 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp, 2196 struct mlxsw_sp_neigh_entry *neigh_entry) 2197 { 2198 struct devlink *devlink; 2199 const char *table_name; 2200 2201 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) { 2202 case AF_INET: 2203 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4; 2204 break; 2205 case AF_INET6: 2206 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6; 2207 break; 2208 default: 2209 WARN_ON(1); 2210 return false; 2211 } 2212 2213 devlink = priv_to_devlink(mlxsw_sp->core); 2214 return devlink_dpipe_table_counter_enabled(devlink, table_name); 2215 } 2216 2217 static void 2218 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2219 struct mlxsw_sp_neigh_entry *neigh_entry) 2220 { 2221 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry)) 2222 return; 2223 2224 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index)) 2225 return; 2226 2227 neigh_entry->counter_valid = true; 2228 } 2229 2230 static void 2231 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp, 2232 struct mlxsw_sp_neigh_entry *neigh_entry) 2233 { 2234 if (!neigh_entry->counter_valid) 2235 return; 2236 mlxsw_sp_flow_counter_free(mlxsw_sp, 2237 neigh_entry->counter_index); 2238 neigh_entry->counter_valid = false; 2239 } 2240 2241 static struct mlxsw_sp_neigh_entry * 2242 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2243 { 2244 struct mlxsw_sp_neigh_entry *neigh_entry; 2245 struct mlxsw_sp_rif *rif; 2246 int err; 2247 2248 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev); 2249 if (!rif) 2250 return ERR_PTR(-EINVAL); 2251 2252 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index); 2253 if (!neigh_entry) 2254 return ERR_PTR(-ENOMEM); 2255 2256 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 2257 if (err) 2258 goto err_neigh_entry_insert; 2259 2260 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2261 atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count); 2262 list_add(&neigh_entry->rif_list_node, &rif->neigh_list); 2263 2264 return neigh_entry; 2265 2266 err_neigh_entry_insert: 2267 mlxsw_sp_neigh_entry_free(neigh_entry); 2268 return ERR_PTR(err); 2269 } 2270 2271 static void 2272 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp, 2273 struct mlxsw_sp_neigh_entry *neigh_entry) 2274 { 2275 list_del(&neigh_entry->rif_list_node); 2276 atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count); 2277 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2278 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 2279 mlxsw_sp_neigh_entry_free(neigh_entry); 2280 } 2281 2282 static struct mlxsw_sp_neigh_entry * 2283 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n) 2284 { 2285 struct mlxsw_sp_neigh_key key; 2286 2287 key.n = n; 2288 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht, 2289 &key, mlxsw_sp_neigh_ht_params); 2290 } 2291 2292 static void 2293 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp) 2294 { 2295 unsigned long interval; 2296 2297 #if IS_ENABLED(CONFIG_IPV6) 2298 interval = min_t(unsigned long, 2299 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME), 2300 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME)); 2301 #else 2302 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 2303 #endif 2304 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval); 2305 } 2306 2307 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2308 char *rauhtd_pl, 2309 int ent_index) 2310 { 2311 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 2312 struct net_device *dev; 2313 struct neighbour *n; 2314 __be32 dipn; 2315 u32 dip; 2316 u16 rif; 2317 2318 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip); 2319 2320 if (WARN_ON_ONCE(rif >= max_rifs)) 2321 return; 2322 if (!mlxsw_sp->router->rifs[rif]) { 2323 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2324 return; 2325 } 2326 2327 dipn = htonl(dip); 2328 dev = mlxsw_sp->router->rifs[rif]->dev; 2329 n = neigh_lookup(&arp_tbl, &dipn, dev); 2330 if (!n) 2331 return; 2332 2333 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip); 2334 neigh_event_send(n, NULL); 2335 neigh_release(n); 2336 } 2337 2338 #if IS_ENABLED(CONFIG_IPV6) 2339 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2340 char *rauhtd_pl, 2341 int rec_index) 2342 { 2343 struct net_device *dev; 2344 struct neighbour *n; 2345 struct in6_addr dip; 2346 u16 rif; 2347 2348 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif, 2349 (char *) &dip); 2350 2351 if (!mlxsw_sp->router->rifs[rif]) { 2352 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n"); 2353 return; 2354 } 2355 2356 dev = mlxsw_sp->router->rifs[rif]->dev; 2357 n = neigh_lookup(&nd_tbl, &dip, dev); 2358 if (!n) 2359 return; 2360 2361 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip); 2362 neigh_event_send(n, NULL); 2363 neigh_release(n); 2364 } 2365 #else 2366 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2367 char *rauhtd_pl, 2368 int rec_index) 2369 { 2370 } 2371 #endif 2372 2373 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp, 2374 char *rauhtd_pl, 2375 int rec_index) 2376 { 2377 u8 num_entries; 2378 int i; 2379 2380 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2381 rec_index); 2382 /* Hardware starts counting at 0, so add 1. */ 2383 num_entries++; 2384 2385 /* Each record consists of several neighbour entries. */ 2386 for (i = 0; i < num_entries; i++) { 2387 int ent_index; 2388 2389 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i; 2390 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl, 2391 ent_index); 2392 } 2393 2394 } 2395 2396 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp, 2397 char *rauhtd_pl, 2398 int rec_index) 2399 { 2400 /* One record contains one entry. */ 2401 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl, 2402 rec_index); 2403 } 2404 2405 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, 2406 char *rauhtd_pl, int rec_index) 2407 { 2408 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) { 2409 case MLXSW_REG_RAUHTD_TYPE_IPV4: 2410 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl, 2411 rec_index); 2412 break; 2413 case MLXSW_REG_RAUHTD_TYPE_IPV6: 2414 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl, 2415 rec_index); 2416 break; 2417 } 2418 } 2419 2420 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) 2421 { 2422 u8 num_rec, last_rec_index, num_entries; 2423 2424 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2425 last_rec_index = num_rec - 1; 2426 2427 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) 2428 return false; 2429 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == 2430 MLXSW_REG_RAUHTD_TYPE_IPV6) 2431 return true; 2432 2433 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, 2434 last_rec_index); 2435 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) 2436 return true; 2437 return false; 2438 } 2439 2440 static int 2441 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp, 2442 char *rauhtd_pl, 2443 enum mlxsw_reg_rauhtd_type type) 2444 { 2445 int i, num_rec; 2446 int err; 2447 2448 /* Ensure the RIF we read from the device does not change mid-dump. */ 2449 mutex_lock(&mlxsw_sp->router->lock); 2450 do { 2451 mlxsw_reg_rauhtd_pack(rauhtd_pl, type); 2452 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd), 2453 rauhtd_pl); 2454 if (err) { 2455 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n"); 2456 break; 2457 } 2458 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); 2459 for (i = 0; i < num_rec; i++) 2460 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, 2461 i); 2462 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); 2463 mutex_unlock(&mlxsw_sp->router->lock); 2464 2465 return err; 2466 } 2467 2468 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) 2469 { 2470 enum mlxsw_reg_rauhtd_type type; 2471 char *rauhtd_pl; 2472 int err; 2473 2474 if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count)) 2475 return 0; 2476 2477 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL); 2478 if (!rauhtd_pl) 2479 return -ENOMEM; 2480 2481 type = MLXSW_REG_RAUHTD_TYPE_IPV4; 2482 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2483 if (err) 2484 goto out; 2485 2486 type = MLXSW_REG_RAUHTD_TYPE_IPV6; 2487 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type); 2488 out: 2489 kfree(rauhtd_pl); 2490 return err; 2491 } 2492 2493 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp) 2494 { 2495 struct mlxsw_sp_neigh_entry *neigh_entry; 2496 2497 mutex_lock(&mlxsw_sp->router->lock); 2498 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list, 2499 nexthop_neighs_list_node) 2500 /* If this neigh have nexthops, make the kernel think this neigh 2501 * is active regardless of the traffic. 2502 */ 2503 neigh_event_send(neigh_entry->key.n, NULL); 2504 mutex_unlock(&mlxsw_sp->router->lock); 2505 } 2506 2507 static void 2508 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp) 2509 { 2510 unsigned long interval = mlxsw_sp->router->neighs_update.interval; 2511 2512 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 2513 msecs_to_jiffies(interval)); 2514 } 2515 2516 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work) 2517 { 2518 struct mlxsw_sp_router *router; 2519 int err; 2520 2521 router = container_of(work, struct mlxsw_sp_router, 2522 neighs_update.dw.work); 2523 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp); 2524 if (err) 2525 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity"); 2526 2527 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp); 2528 2529 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp); 2530 } 2531 2532 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work) 2533 { 2534 struct mlxsw_sp_neigh_entry *neigh_entry; 2535 struct mlxsw_sp_router *router; 2536 2537 router = container_of(work, struct mlxsw_sp_router, 2538 nexthop_probe_dw.work); 2539 /* Iterate over nexthop neighbours, find those who are unresolved and 2540 * send arp on them. This solves the chicken-egg problem when 2541 * the nexthop wouldn't get offloaded until the neighbor is resolved 2542 * but it wouldn't get resolved ever in case traffic is flowing in HW 2543 * using different nexthop. 2544 */ 2545 mutex_lock(&router->lock); 2546 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list, 2547 nexthop_neighs_list_node) 2548 if (!neigh_entry->connected) 2549 neigh_event_send(neigh_entry->key.n, NULL); 2550 mutex_unlock(&router->lock); 2551 2552 mlxsw_core_schedule_dw(&router->nexthop_probe_dw, 2553 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL); 2554 } 2555 2556 static void 2557 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 2558 struct mlxsw_sp_neigh_entry *neigh_entry, 2559 bool removing, bool dead); 2560 2561 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding) 2562 { 2563 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD : 2564 MLXSW_REG_RAUHT_OP_WRITE_DELETE; 2565 } 2566 2567 static int 2568 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp, 2569 struct mlxsw_sp_neigh_entry *neigh_entry, 2570 enum mlxsw_reg_rauht_op op) 2571 { 2572 struct neighbour *n = neigh_entry->key.n; 2573 u32 dip = ntohl(*((__be32 *) n->primary_key)); 2574 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2575 2576 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2577 dip); 2578 if (neigh_entry->counter_valid) 2579 mlxsw_reg_rauht_pack_counter(rauht_pl, 2580 neigh_entry->counter_index); 2581 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2582 } 2583 2584 static int 2585 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp, 2586 struct mlxsw_sp_neigh_entry *neigh_entry, 2587 enum mlxsw_reg_rauht_op op) 2588 { 2589 struct neighbour *n = neigh_entry->key.n; 2590 char rauht_pl[MLXSW_REG_RAUHT_LEN]; 2591 const char *dip = n->primary_key; 2592 2593 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha, 2594 dip); 2595 if (neigh_entry->counter_valid) 2596 mlxsw_reg_rauht_pack_counter(rauht_pl, 2597 neigh_entry->counter_index); 2598 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl); 2599 } 2600 2601 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry) 2602 { 2603 struct neighbour *n = neigh_entry->key.n; 2604 2605 /* Packets with a link-local destination address are trapped 2606 * after LPM lookup and never reach the neighbour table, so 2607 * there is no need to program such neighbours to the device. 2608 */ 2609 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) & 2610 IPV6_ADDR_LINKLOCAL) 2611 return true; 2612 return false; 2613 } 2614 2615 static void 2616 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp, 2617 struct mlxsw_sp_neigh_entry *neigh_entry, 2618 bool adding) 2619 { 2620 enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding); 2621 int err; 2622 2623 if (!adding && !neigh_entry->connected) 2624 return; 2625 neigh_entry->connected = adding; 2626 if (neigh_entry->key.n->tbl->family == AF_INET) { 2627 err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry, 2628 op); 2629 if (err) 2630 return; 2631 } else if (neigh_entry->key.n->tbl->family == AF_INET6) { 2632 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry)) 2633 return; 2634 err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry, 2635 op); 2636 if (err) 2637 return; 2638 } else { 2639 WARN_ON_ONCE(1); 2640 return; 2641 } 2642 2643 if (adding) 2644 neigh_entry->key.n->flags |= NTF_OFFLOADED; 2645 else 2646 neigh_entry->key.n->flags &= ~NTF_OFFLOADED; 2647 } 2648 2649 void 2650 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp, 2651 struct mlxsw_sp_neigh_entry *neigh_entry, 2652 bool adding) 2653 { 2654 if (adding) 2655 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry); 2656 else 2657 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry); 2658 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true); 2659 } 2660 2661 struct mlxsw_sp_netevent_work { 2662 struct work_struct work; 2663 struct mlxsw_sp *mlxsw_sp; 2664 struct neighbour *n; 2665 }; 2666 2667 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work) 2668 { 2669 struct mlxsw_sp_netevent_work *net_work = 2670 container_of(work, struct mlxsw_sp_netevent_work, work); 2671 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2672 struct mlxsw_sp_neigh_entry *neigh_entry; 2673 struct neighbour *n = net_work->n; 2674 unsigned char ha[ETH_ALEN]; 2675 bool entry_connected; 2676 u8 nud_state, dead; 2677 2678 /* If these parameters are changed after we release the lock, 2679 * then we are guaranteed to receive another event letting us 2680 * know about it. 2681 */ 2682 read_lock_bh(&n->lock); 2683 memcpy(ha, n->ha, ETH_ALEN); 2684 nud_state = n->nud_state; 2685 dead = n->dead; 2686 read_unlock_bh(&n->lock); 2687 2688 mutex_lock(&mlxsw_sp->router->lock); 2689 mlxsw_sp_span_respin(mlxsw_sp); 2690 2691 entry_connected = nud_state & NUD_VALID && !dead; 2692 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 2693 if (!entry_connected && !neigh_entry) 2694 goto out; 2695 if (!neigh_entry) { 2696 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 2697 if (IS_ERR(neigh_entry)) 2698 goto out; 2699 } 2700 2701 if (neigh_entry->connected && entry_connected && 2702 !memcmp(neigh_entry->ha, ha, ETH_ALEN)) 2703 goto out; 2704 2705 memcpy(neigh_entry->ha, ha, ETH_ALEN); 2706 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected); 2707 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected, 2708 dead); 2709 2710 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 2711 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2712 2713 out: 2714 mutex_unlock(&mlxsw_sp->router->lock); 2715 neigh_release(n); 2716 kfree(net_work); 2717 } 2718 2719 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp); 2720 2721 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work) 2722 { 2723 struct mlxsw_sp_netevent_work *net_work = 2724 container_of(work, struct mlxsw_sp_netevent_work, work); 2725 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2726 2727 mlxsw_sp_mp_hash_init(mlxsw_sp); 2728 kfree(net_work); 2729 } 2730 2731 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp); 2732 2733 static void mlxsw_sp_router_update_priority_work(struct work_struct *work) 2734 { 2735 struct mlxsw_sp_netevent_work *net_work = 2736 container_of(work, struct mlxsw_sp_netevent_work, work); 2737 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp; 2738 2739 __mlxsw_sp_router_init(mlxsw_sp); 2740 kfree(net_work); 2741 } 2742 2743 static int mlxsw_sp_router_schedule_work(struct net *net, 2744 struct notifier_block *nb, 2745 void (*cb)(struct work_struct *)) 2746 { 2747 struct mlxsw_sp_netevent_work *net_work; 2748 struct mlxsw_sp_router *router; 2749 2750 router = container_of(nb, struct mlxsw_sp_router, netevent_nb); 2751 if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp))) 2752 return NOTIFY_DONE; 2753 2754 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2755 if (!net_work) 2756 return NOTIFY_BAD; 2757 2758 INIT_WORK(&net_work->work, cb); 2759 net_work->mlxsw_sp = router->mlxsw_sp; 2760 mlxsw_core_schedule_work(&net_work->work); 2761 return NOTIFY_DONE; 2762 } 2763 2764 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb, 2765 unsigned long event, void *ptr) 2766 { 2767 struct mlxsw_sp_netevent_work *net_work; 2768 struct mlxsw_sp_port *mlxsw_sp_port; 2769 struct mlxsw_sp *mlxsw_sp; 2770 unsigned long interval; 2771 struct neigh_parms *p; 2772 struct neighbour *n; 2773 2774 switch (event) { 2775 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 2776 p = ptr; 2777 2778 /* We don't care about changes in the default table. */ 2779 if (!p->dev || (p->tbl->family != AF_INET && 2780 p->tbl->family != AF_INET6)) 2781 return NOTIFY_DONE; 2782 2783 /* We are in atomic context and can't take RTNL mutex, 2784 * so use RCU variant to walk the device chain. 2785 */ 2786 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev); 2787 if (!mlxsw_sp_port) 2788 return NOTIFY_DONE; 2789 2790 mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2791 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME)); 2792 mlxsw_sp->router->neighs_update.interval = interval; 2793 2794 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2795 break; 2796 case NETEVENT_NEIGH_UPDATE: 2797 n = ptr; 2798 2799 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6) 2800 return NOTIFY_DONE; 2801 2802 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev); 2803 if (!mlxsw_sp_port) 2804 return NOTIFY_DONE; 2805 2806 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC); 2807 if (!net_work) { 2808 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2809 return NOTIFY_BAD; 2810 } 2811 2812 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work); 2813 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 2814 net_work->n = n; 2815 2816 /* Take a reference to ensure the neighbour won't be 2817 * destructed until we drop the reference in delayed 2818 * work. 2819 */ 2820 neigh_clone(n); 2821 mlxsw_core_schedule_work(&net_work->work); 2822 mlxsw_sp_port_dev_put(mlxsw_sp_port); 2823 break; 2824 case NETEVENT_IPV4_MPATH_HASH_UPDATE: 2825 case NETEVENT_IPV6_MPATH_HASH_UPDATE: 2826 return mlxsw_sp_router_schedule_work(ptr, nb, 2827 mlxsw_sp_router_mp_hash_event_work); 2828 2829 case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE: 2830 return mlxsw_sp_router_schedule_work(ptr, nb, 2831 mlxsw_sp_router_update_priority_work); 2832 } 2833 2834 return NOTIFY_DONE; 2835 } 2836 2837 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp) 2838 { 2839 int err; 2840 2841 err = rhashtable_init(&mlxsw_sp->router->neigh_ht, 2842 &mlxsw_sp_neigh_ht_params); 2843 if (err) 2844 return err; 2845 2846 /* Initialize the polling interval according to the default 2847 * table. 2848 */ 2849 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp); 2850 2851 /* Create the delayed works for the activity_update */ 2852 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw, 2853 mlxsw_sp_router_neighs_update_work); 2854 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw, 2855 mlxsw_sp_router_probe_unresolved_nexthops); 2856 atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0); 2857 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0); 2858 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0); 2859 return 0; 2860 } 2861 2862 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp) 2863 { 2864 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw); 2865 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw); 2866 rhashtable_destroy(&mlxsw_sp->router->neigh_ht); 2867 } 2868 2869 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 2870 struct mlxsw_sp_rif *rif) 2871 { 2872 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp; 2873 2874 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list, 2875 rif_list_node) { 2876 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false); 2877 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 2878 } 2879 } 2880 2881 enum mlxsw_sp_nexthop_type { 2882 MLXSW_SP_NEXTHOP_TYPE_ETH, 2883 MLXSW_SP_NEXTHOP_TYPE_IPIP, 2884 }; 2885 2886 enum mlxsw_sp_nexthop_action { 2887 /* Nexthop forwards packets to an egress RIF */ 2888 MLXSW_SP_NEXTHOP_ACTION_FORWARD, 2889 /* Nexthop discards packets */ 2890 MLXSW_SP_NEXTHOP_ACTION_DISCARD, 2891 /* Nexthop traps packets */ 2892 MLXSW_SP_NEXTHOP_ACTION_TRAP, 2893 }; 2894 2895 struct mlxsw_sp_nexthop_key { 2896 struct fib_nh *fib_nh; 2897 }; 2898 2899 struct mlxsw_sp_nexthop { 2900 struct list_head neigh_list_node; /* member of neigh entry list */ 2901 struct list_head rif_list_node; 2902 struct list_head router_list_node; 2903 struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group 2904 * this nexthop belongs to 2905 */ 2906 struct rhash_head ht_node; 2907 struct neigh_table *neigh_tbl; 2908 struct mlxsw_sp_nexthop_key key; 2909 unsigned char gw_addr[sizeof(struct in6_addr)]; 2910 int ifindex; 2911 int nh_weight; 2912 int norm_nh_weight; 2913 int num_adj_entries; 2914 struct mlxsw_sp_rif *rif; 2915 u8 should_offload:1, /* set indicates this nexthop should be written 2916 * to the adjacency table. 2917 */ 2918 offloaded:1, /* set indicates this nexthop was written to the 2919 * adjacency table. 2920 */ 2921 update:1; /* set indicates this nexthop should be updated in the 2922 * adjacency table (f.e., its MAC changed). 2923 */ 2924 enum mlxsw_sp_nexthop_action action; 2925 enum mlxsw_sp_nexthop_type type; 2926 union { 2927 struct mlxsw_sp_neigh_entry *neigh_entry; 2928 struct mlxsw_sp_ipip_entry *ipip_entry; 2929 }; 2930 unsigned int counter_index; 2931 bool counter_valid; 2932 }; 2933 2934 enum mlxsw_sp_nexthop_group_type { 2935 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4, 2936 MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6, 2937 MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ, 2938 }; 2939 2940 struct mlxsw_sp_nexthop_group_info { 2941 struct mlxsw_sp_nexthop_group *nh_grp; 2942 u32 adj_index; 2943 u16 ecmp_size; 2944 u16 count; 2945 int sum_norm_weight; 2946 u8 adj_index_valid:1, 2947 gateway:1, /* routes using the group use a gateway */ 2948 is_resilient:1; 2949 struct list_head list; /* member in nh_res_grp_list */ 2950 struct mlxsw_sp_nexthop nexthops[0]; 2951 #define nh_rif nexthops[0].rif 2952 }; 2953 2954 struct mlxsw_sp_nexthop_group_vr_key { 2955 u16 vr_id; 2956 enum mlxsw_sp_l3proto proto; 2957 }; 2958 2959 struct mlxsw_sp_nexthop_group_vr_entry { 2960 struct list_head list; /* member in vr_list */ 2961 struct rhash_head ht_node; /* member in vr_ht */ 2962 refcount_t ref_count; 2963 struct mlxsw_sp_nexthop_group_vr_key key; 2964 }; 2965 2966 struct mlxsw_sp_nexthop_group { 2967 struct rhash_head ht_node; 2968 struct list_head fib_list; /* list of fib entries that use this group */ 2969 union { 2970 struct { 2971 struct fib_info *fi; 2972 } ipv4; 2973 struct { 2974 u32 id; 2975 } obj; 2976 }; 2977 struct mlxsw_sp_nexthop_group_info *nhgi; 2978 struct list_head vr_list; 2979 struct rhashtable vr_ht; 2980 enum mlxsw_sp_nexthop_group_type type; 2981 bool can_destroy; 2982 }; 2983 2984 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp, 2985 struct mlxsw_sp_nexthop *nh) 2986 { 2987 struct devlink *devlink; 2988 2989 devlink = priv_to_devlink(mlxsw_sp->core); 2990 if (!devlink_dpipe_table_counter_enabled(devlink, 2991 MLXSW_SP_DPIPE_TABLE_NAME_ADJ)) 2992 return; 2993 2994 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index)) 2995 return; 2996 2997 nh->counter_valid = true; 2998 } 2999 3000 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp, 3001 struct mlxsw_sp_nexthop *nh) 3002 { 3003 if (!nh->counter_valid) 3004 return; 3005 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index); 3006 nh->counter_valid = false; 3007 } 3008 3009 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp, 3010 struct mlxsw_sp_nexthop *nh, u64 *p_counter) 3011 { 3012 if (!nh->counter_valid) 3013 return -EINVAL; 3014 3015 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index, 3016 p_counter, NULL); 3017 } 3018 3019 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router, 3020 struct mlxsw_sp_nexthop *nh) 3021 { 3022 if (!nh) { 3023 if (list_empty(&router->nexthop_list)) 3024 return NULL; 3025 else 3026 return list_first_entry(&router->nexthop_list, 3027 typeof(*nh), router_list_node); 3028 } 3029 if (list_is_last(&nh->router_list_node, &router->nexthop_list)) 3030 return NULL; 3031 return list_next_entry(nh, router_list_node); 3032 } 3033 3034 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh) 3035 { 3036 return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3037 } 3038 3039 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh) 3040 { 3041 if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH || 3042 !mlxsw_sp_nexthop_is_forward(nh)) 3043 return NULL; 3044 return nh->neigh_entry->ha; 3045 } 3046 3047 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index, 3048 u32 *p_adj_size, u32 *p_adj_hash_index) 3049 { 3050 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3051 u32 adj_hash_index = 0; 3052 int i; 3053 3054 if (!nh->offloaded || !nhgi->adj_index_valid) 3055 return -EINVAL; 3056 3057 *p_adj_index = nhgi->adj_index; 3058 *p_adj_size = nhgi->ecmp_size; 3059 3060 for (i = 0; i < nhgi->count; i++) { 3061 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3062 3063 if (nh_iter == nh) 3064 break; 3065 if (nh_iter->offloaded) 3066 adj_hash_index += nh_iter->num_adj_entries; 3067 } 3068 3069 *p_adj_hash_index = adj_hash_index; 3070 return 0; 3071 } 3072 3073 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh) 3074 { 3075 return nh->rif; 3076 } 3077 3078 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh) 3079 { 3080 struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi; 3081 int i; 3082 3083 for (i = 0; i < nhgi->count; i++) { 3084 struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i]; 3085 3086 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP) 3087 return true; 3088 } 3089 return false; 3090 } 3091 3092 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = { 3093 .key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key), 3094 .head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node), 3095 .key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key), 3096 .automatic_shrinking = true, 3097 }; 3098 3099 static struct mlxsw_sp_nexthop_group_vr_entry * 3100 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp, 3101 const struct mlxsw_sp_fib *fib) 3102 { 3103 struct mlxsw_sp_nexthop_group_vr_key key; 3104 3105 memset(&key, 0, sizeof(key)); 3106 key.vr_id = fib->vr->id; 3107 key.proto = fib->proto; 3108 return rhashtable_lookup_fast(&nh_grp->vr_ht, &key, 3109 mlxsw_sp_nexthop_group_vr_ht_params); 3110 } 3111 3112 static int 3113 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp, 3114 const struct mlxsw_sp_fib *fib) 3115 { 3116 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3117 int err; 3118 3119 vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL); 3120 if (!vr_entry) 3121 return -ENOMEM; 3122 3123 vr_entry->key.vr_id = fib->vr->id; 3124 vr_entry->key.proto = fib->proto; 3125 refcount_set(&vr_entry->ref_count, 1); 3126 3127 err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3128 mlxsw_sp_nexthop_group_vr_ht_params); 3129 if (err) 3130 goto err_hashtable_insert; 3131 3132 list_add(&vr_entry->list, &nh_grp->vr_list); 3133 3134 return 0; 3135 3136 err_hashtable_insert: 3137 kfree(vr_entry); 3138 return err; 3139 } 3140 3141 static void 3142 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp, 3143 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry) 3144 { 3145 list_del(&vr_entry->list); 3146 rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node, 3147 mlxsw_sp_nexthop_group_vr_ht_params); 3148 kfree(vr_entry); 3149 } 3150 3151 static int 3152 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp, 3153 const struct mlxsw_sp_fib *fib) 3154 { 3155 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3156 3157 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3158 if (vr_entry) { 3159 refcount_inc(&vr_entry->ref_count); 3160 return 0; 3161 } 3162 3163 return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib); 3164 } 3165 3166 static void 3167 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp, 3168 const struct mlxsw_sp_fib *fib) 3169 { 3170 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3171 3172 vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib); 3173 if (WARN_ON_ONCE(!vr_entry)) 3174 return; 3175 3176 if (!refcount_dec_and_test(&vr_entry->ref_count)) 3177 return; 3178 3179 mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry); 3180 } 3181 3182 struct mlxsw_sp_nexthop_group_cmp_arg { 3183 enum mlxsw_sp_nexthop_group_type type; 3184 union { 3185 struct fib_info *fi; 3186 struct mlxsw_sp_fib6_entry *fib6_entry; 3187 u32 id; 3188 }; 3189 }; 3190 3191 static bool 3192 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp, 3193 const struct in6_addr *gw, int ifindex, 3194 int weight) 3195 { 3196 int i; 3197 3198 for (i = 0; i < nh_grp->nhgi->count; i++) { 3199 const struct mlxsw_sp_nexthop *nh; 3200 3201 nh = &nh_grp->nhgi->nexthops[i]; 3202 if (nh->ifindex == ifindex && nh->nh_weight == weight && 3203 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr)) 3204 return true; 3205 } 3206 3207 return false; 3208 } 3209 3210 static bool 3211 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp, 3212 const struct mlxsw_sp_fib6_entry *fib6_entry) 3213 { 3214 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3215 3216 if (nh_grp->nhgi->count != fib6_entry->nrt6) 3217 return false; 3218 3219 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3220 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3221 struct in6_addr *gw; 3222 int ifindex, weight; 3223 3224 ifindex = fib6_nh->fib_nh_dev->ifindex; 3225 weight = fib6_nh->fib_nh_weight; 3226 gw = &fib6_nh->fib_nh_gw6; 3227 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex, 3228 weight)) 3229 return false; 3230 } 3231 3232 return true; 3233 } 3234 3235 static int 3236 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr) 3237 { 3238 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key; 3239 const struct mlxsw_sp_nexthop_group *nh_grp = ptr; 3240 3241 if (nh_grp->type != cmp_arg->type) 3242 return 1; 3243 3244 switch (cmp_arg->type) { 3245 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3246 return cmp_arg->fi != nh_grp->ipv4.fi; 3247 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3248 return !mlxsw_sp_nexthop6_group_cmp(nh_grp, 3249 cmp_arg->fib6_entry); 3250 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3251 return cmp_arg->id != nh_grp->obj.id; 3252 default: 3253 WARN_ON(1); 3254 return 1; 3255 } 3256 } 3257 3258 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed) 3259 { 3260 const struct mlxsw_sp_nexthop_group *nh_grp = data; 3261 const struct mlxsw_sp_nexthop *nh; 3262 struct fib_info *fi; 3263 unsigned int val; 3264 int i; 3265 3266 switch (nh_grp->type) { 3267 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3268 fi = nh_grp->ipv4.fi; 3269 return jhash(&fi, sizeof(fi), seed); 3270 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3271 val = nh_grp->nhgi->count; 3272 for (i = 0; i < nh_grp->nhgi->count; i++) { 3273 nh = &nh_grp->nhgi->nexthops[i]; 3274 val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed); 3275 val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed); 3276 } 3277 return jhash(&val, sizeof(val), seed); 3278 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3279 return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed); 3280 default: 3281 WARN_ON(1); 3282 return 0; 3283 } 3284 } 3285 3286 static u32 3287 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed) 3288 { 3289 unsigned int val = fib6_entry->nrt6; 3290 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3291 3292 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3293 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3294 struct net_device *dev = fib6_nh->fib_nh_dev; 3295 struct in6_addr *gw = &fib6_nh->fib_nh_gw6; 3296 3297 val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed); 3298 val ^= jhash(gw, sizeof(*gw), seed); 3299 } 3300 3301 return jhash(&val, sizeof(val), seed); 3302 } 3303 3304 static u32 3305 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed) 3306 { 3307 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data; 3308 3309 switch (cmp_arg->type) { 3310 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3311 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed); 3312 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3313 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed); 3314 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3315 return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed); 3316 default: 3317 WARN_ON(1); 3318 return 0; 3319 } 3320 } 3321 3322 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = { 3323 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node), 3324 .hashfn = mlxsw_sp_nexthop_group_hash, 3325 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj, 3326 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp, 3327 }; 3328 3329 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp, 3330 struct mlxsw_sp_nexthop_group *nh_grp) 3331 { 3332 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3333 !nh_grp->nhgi->gateway) 3334 return 0; 3335 3336 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht, 3337 &nh_grp->ht_node, 3338 mlxsw_sp_nexthop_group_ht_params); 3339 } 3340 3341 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp, 3342 struct mlxsw_sp_nexthop_group *nh_grp) 3343 { 3344 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 && 3345 !nh_grp->nhgi->gateway) 3346 return; 3347 3348 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht, 3349 &nh_grp->ht_node, 3350 mlxsw_sp_nexthop_group_ht_params); 3351 } 3352 3353 static struct mlxsw_sp_nexthop_group * 3354 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp, 3355 struct fib_info *fi) 3356 { 3357 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3358 3359 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 3360 cmp_arg.fi = fi; 3361 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3362 &cmp_arg, 3363 mlxsw_sp_nexthop_group_ht_params); 3364 } 3365 3366 static struct mlxsw_sp_nexthop_group * 3367 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp, 3368 struct mlxsw_sp_fib6_entry *fib6_entry) 3369 { 3370 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 3371 3372 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 3373 cmp_arg.fib6_entry = fib6_entry; 3374 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 3375 &cmp_arg, 3376 mlxsw_sp_nexthop_group_ht_params); 3377 } 3378 3379 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = { 3380 .key_offset = offsetof(struct mlxsw_sp_nexthop, key), 3381 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node), 3382 .key_len = sizeof(struct mlxsw_sp_nexthop_key), 3383 }; 3384 3385 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp, 3386 struct mlxsw_sp_nexthop *nh) 3387 { 3388 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht, 3389 &nh->ht_node, mlxsw_sp_nexthop_ht_params); 3390 } 3391 3392 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp, 3393 struct mlxsw_sp_nexthop *nh) 3394 { 3395 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node, 3396 mlxsw_sp_nexthop_ht_params); 3397 } 3398 3399 static struct mlxsw_sp_nexthop * 3400 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp, 3401 struct mlxsw_sp_nexthop_key key) 3402 { 3403 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key, 3404 mlxsw_sp_nexthop_ht_params); 3405 } 3406 3407 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp, 3408 enum mlxsw_sp_l3proto proto, 3409 u16 vr_id, 3410 u32 adj_index, u16 ecmp_size, 3411 u32 new_adj_index, 3412 u16 new_ecmp_size) 3413 { 3414 char raleu_pl[MLXSW_REG_RALEU_LEN]; 3415 3416 mlxsw_reg_raleu_pack(raleu_pl, 3417 (enum mlxsw_reg_ralxx_protocol) proto, vr_id, 3418 adj_index, ecmp_size, new_adj_index, 3419 new_ecmp_size); 3420 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl); 3421 } 3422 3423 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp, 3424 struct mlxsw_sp_nexthop_group *nh_grp, 3425 u32 old_adj_index, u16 old_ecmp_size) 3426 { 3427 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3428 struct mlxsw_sp_nexthop_group_vr_entry *vr_entry; 3429 int err; 3430 3431 list_for_each_entry(vr_entry, &nh_grp->vr_list, list) { 3432 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, 3433 vr_entry->key.proto, 3434 vr_entry->key.vr_id, 3435 old_adj_index, 3436 old_ecmp_size, 3437 nhgi->adj_index, 3438 nhgi->ecmp_size); 3439 if (err) 3440 goto err_mass_update_vr; 3441 } 3442 return 0; 3443 3444 err_mass_update_vr: 3445 list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list) 3446 mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto, 3447 vr_entry->key.vr_id, 3448 nhgi->adj_index, 3449 nhgi->ecmp_size, 3450 old_adj_index, old_ecmp_size); 3451 return err; 3452 } 3453 3454 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, 3455 u32 adj_index, 3456 struct mlxsw_sp_nexthop *nh, 3457 bool force, char *ratr_pl) 3458 { 3459 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 3460 enum mlxsw_reg_ratr_op op; 3461 u16 rif_index; 3462 3463 rif_index = nh->rif ? nh->rif->rif_index : 3464 mlxsw_sp->router->lb_rif_index; 3465 op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY : 3466 MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY; 3467 mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET, 3468 adj_index, rif_index); 3469 switch (nh->action) { 3470 case MLXSW_SP_NEXTHOP_ACTION_FORWARD: 3471 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha); 3472 break; 3473 case MLXSW_SP_NEXTHOP_ACTION_DISCARD: 3474 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3475 MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS); 3476 break; 3477 case MLXSW_SP_NEXTHOP_ACTION_TRAP: 3478 mlxsw_reg_ratr_trap_action_set(ratr_pl, 3479 MLXSW_REG_RATR_TRAP_ACTION_TRAP); 3480 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 3481 break; 3482 default: 3483 WARN_ON_ONCE(1); 3484 return -EINVAL; 3485 } 3486 if (nh->counter_valid) 3487 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true); 3488 else 3489 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false); 3490 3491 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 3492 } 3493 3494 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3495 struct mlxsw_sp_nexthop *nh, bool force, 3496 char *ratr_pl) 3497 { 3498 int i; 3499 3500 for (i = 0; i < nh->num_adj_entries; i++) { 3501 int err; 3502 3503 err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i, 3504 nh, force, ratr_pl); 3505 if (err) 3506 return err; 3507 } 3508 3509 return 0; 3510 } 3511 3512 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3513 u32 adj_index, 3514 struct mlxsw_sp_nexthop *nh, 3515 bool force, char *ratr_pl) 3516 { 3517 const struct mlxsw_sp_ipip_ops *ipip_ops; 3518 3519 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt]; 3520 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry, 3521 force, ratr_pl); 3522 } 3523 3524 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp, 3525 u32 adj_index, 3526 struct mlxsw_sp_nexthop *nh, bool force, 3527 char *ratr_pl) 3528 { 3529 int i; 3530 3531 for (i = 0; i < nh->num_adj_entries; i++) { 3532 int err; 3533 3534 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i, 3535 nh, force, ratr_pl); 3536 if (err) 3537 return err; 3538 } 3539 3540 return 0; 3541 } 3542 3543 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index, 3544 struct mlxsw_sp_nexthop *nh, bool force, 3545 char *ratr_pl) 3546 { 3547 /* When action is discard or trap, the nexthop must be 3548 * programmed as an Ethernet nexthop. 3549 */ 3550 if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH || 3551 nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD || 3552 nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3553 return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh, 3554 force, ratr_pl); 3555 else 3556 return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh, 3557 force, ratr_pl); 3558 } 3559 3560 static int 3561 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp, 3562 struct mlxsw_sp_nexthop_group_info *nhgi, 3563 bool reallocate) 3564 { 3565 char ratr_pl[MLXSW_REG_RATR_LEN]; 3566 u32 adj_index = nhgi->adj_index; /* base */ 3567 struct mlxsw_sp_nexthop *nh; 3568 int i; 3569 3570 for (i = 0; i < nhgi->count; i++) { 3571 nh = &nhgi->nexthops[i]; 3572 3573 if (!nh->should_offload) { 3574 nh->offloaded = 0; 3575 continue; 3576 } 3577 3578 if (nh->update || reallocate) { 3579 int err = 0; 3580 3581 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, 3582 true, ratr_pl); 3583 if (err) 3584 return err; 3585 nh->update = 0; 3586 nh->offloaded = 1; 3587 } 3588 adj_index += nh->num_adj_entries; 3589 } 3590 return 0; 3591 } 3592 3593 static int 3594 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp, 3595 struct mlxsw_sp_nexthop_group *nh_grp) 3596 { 3597 struct mlxsw_sp_fib_entry *fib_entry; 3598 int err; 3599 3600 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) { 3601 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 3602 if (err) 3603 return err; 3604 } 3605 return 0; 3606 } 3607 3608 struct mlxsw_sp_adj_grp_size_range { 3609 u16 start; /* Inclusive */ 3610 u16 end; /* Inclusive */ 3611 }; 3612 3613 /* Ordered by range start value */ 3614 static const struct mlxsw_sp_adj_grp_size_range 3615 mlxsw_sp1_adj_grp_size_ranges[] = { 3616 { .start = 1, .end = 64 }, 3617 { .start = 512, .end = 512 }, 3618 { .start = 1024, .end = 1024 }, 3619 { .start = 2048, .end = 2048 }, 3620 { .start = 4096, .end = 4096 }, 3621 }; 3622 3623 /* Ordered by range start value */ 3624 static const struct mlxsw_sp_adj_grp_size_range 3625 mlxsw_sp2_adj_grp_size_ranges[] = { 3626 { .start = 1, .end = 128 }, 3627 { .start = 256, .end = 256 }, 3628 { .start = 512, .end = 512 }, 3629 { .start = 1024, .end = 1024 }, 3630 { .start = 2048, .end = 2048 }, 3631 { .start = 4096, .end = 4096 }, 3632 }; 3633 3634 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp, 3635 u16 *p_adj_grp_size) 3636 { 3637 int i; 3638 3639 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 3640 const struct mlxsw_sp_adj_grp_size_range *size_range; 3641 3642 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3643 3644 if (*p_adj_grp_size >= size_range->start && 3645 *p_adj_grp_size <= size_range->end) 3646 return; 3647 3648 if (*p_adj_grp_size <= size_range->end) { 3649 *p_adj_grp_size = size_range->end; 3650 return; 3651 } 3652 } 3653 } 3654 3655 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp, 3656 u16 *p_adj_grp_size, 3657 unsigned int alloc_size) 3658 { 3659 int i; 3660 3661 for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) { 3662 const struct mlxsw_sp_adj_grp_size_range *size_range; 3663 3664 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 3665 3666 if (alloc_size >= size_range->end) { 3667 *p_adj_grp_size = size_range->end; 3668 return; 3669 } 3670 } 3671 } 3672 3673 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp, 3674 u16 *p_adj_grp_size) 3675 { 3676 unsigned int alloc_size; 3677 int err; 3678 3679 /* Round up the requested group size to the next size supported 3680 * by the device and make sure the request can be satisfied. 3681 */ 3682 mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size); 3683 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 3684 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3685 *p_adj_grp_size, &alloc_size); 3686 if (err) 3687 return err; 3688 /* It is possible the allocation results in more allocated 3689 * entries than requested. Try to use as much of them as 3690 * possible. 3691 */ 3692 mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size); 3693 3694 return 0; 3695 } 3696 3697 static void 3698 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi) 3699 { 3700 int i, g = 0, sum_norm_weight = 0; 3701 struct mlxsw_sp_nexthop *nh; 3702 3703 for (i = 0; i < nhgi->count; i++) { 3704 nh = &nhgi->nexthops[i]; 3705 3706 if (!nh->should_offload) 3707 continue; 3708 if (g > 0) 3709 g = gcd(nh->nh_weight, g); 3710 else 3711 g = nh->nh_weight; 3712 } 3713 3714 for (i = 0; i < nhgi->count; i++) { 3715 nh = &nhgi->nexthops[i]; 3716 3717 if (!nh->should_offload) 3718 continue; 3719 nh->norm_nh_weight = nh->nh_weight / g; 3720 sum_norm_weight += nh->norm_nh_weight; 3721 } 3722 3723 nhgi->sum_norm_weight = sum_norm_weight; 3724 } 3725 3726 static void 3727 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi) 3728 { 3729 int i, weight = 0, lower_bound = 0; 3730 int total = nhgi->sum_norm_weight; 3731 u16 ecmp_size = nhgi->ecmp_size; 3732 3733 for (i = 0; i < nhgi->count; i++) { 3734 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 3735 int upper_bound; 3736 3737 if (!nh->should_offload) 3738 continue; 3739 weight += nh->norm_nh_weight; 3740 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total); 3741 nh->num_adj_entries = upper_bound - lower_bound; 3742 lower_bound = upper_bound; 3743 } 3744 } 3745 3746 static struct mlxsw_sp_nexthop * 3747 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 3748 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6); 3749 3750 static void 3751 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3752 struct mlxsw_sp_nexthop_group *nh_grp) 3753 { 3754 int i; 3755 3756 for (i = 0; i < nh_grp->nhgi->count; i++) { 3757 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3758 3759 if (nh->offloaded) 3760 nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3761 else 3762 nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3763 } 3764 } 3765 3766 static void 3767 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp, 3768 struct mlxsw_sp_fib6_entry *fib6_entry) 3769 { 3770 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 3771 3772 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 3773 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 3774 struct mlxsw_sp_nexthop *nh; 3775 3776 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6); 3777 if (nh && nh->offloaded) 3778 fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD; 3779 else 3780 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 3781 } 3782 } 3783 3784 static void 3785 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3786 struct mlxsw_sp_nexthop_group *nh_grp) 3787 { 3788 struct mlxsw_sp_fib6_entry *fib6_entry; 3789 3790 /* Unfortunately, in IPv6 the route and the nexthop are described by 3791 * the same struct, so we need to iterate over all the routes using the 3792 * nexthop group and set / clear the offload indication for them. 3793 */ 3794 list_for_each_entry(fib6_entry, &nh_grp->fib_list, 3795 common.nexthop_group_node) 3796 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 3797 } 3798 3799 static void 3800 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3801 const struct mlxsw_sp_nexthop *nh, 3802 u16 bucket_index) 3803 { 3804 struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp; 3805 bool offload = false, trap = false; 3806 3807 if (nh->offloaded) { 3808 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP) 3809 trap = true; 3810 else 3811 offload = true; 3812 } 3813 nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3814 bucket_index, offload, trap); 3815 } 3816 3817 static void 3818 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3819 struct mlxsw_sp_nexthop_group *nh_grp) 3820 { 3821 int i; 3822 3823 /* Do not update the flags if the nexthop group is being destroyed 3824 * since: 3825 * 1. The nexthop objects is being deleted, in which case the flags are 3826 * irrelevant. 3827 * 2. The nexthop group was replaced by a newer group, in which case 3828 * the flags of the nexthop object were already updated based on the 3829 * new group. 3830 */ 3831 if (nh_grp->can_destroy) 3832 return; 3833 3834 nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 3835 nh_grp->nhgi->adj_index_valid, false); 3836 3837 /* Update flags of individual nexthop buckets in case of a resilient 3838 * nexthop group. 3839 */ 3840 if (!nh_grp->nhgi->is_resilient) 3841 return; 3842 3843 for (i = 0; i < nh_grp->nhgi->count; i++) { 3844 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 3845 3846 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i); 3847 } 3848 } 3849 3850 static void 3851 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp, 3852 struct mlxsw_sp_nexthop_group *nh_grp) 3853 { 3854 switch (nh_grp->type) { 3855 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4: 3856 mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp); 3857 break; 3858 case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6: 3859 mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp); 3860 break; 3861 case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ: 3862 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp); 3863 break; 3864 } 3865 } 3866 3867 static int 3868 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp, 3869 struct mlxsw_sp_nexthop_group *nh_grp) 3870 { 3871 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 3872 u16 ecmp_size, old_ecmp_size; 3873 struct mlxsw_sp_nexthop *nh; 3874 bool offload_change = false; 3875 u32 adj_index; 3876 bool old_adj_index_valid; 3877 u32 old_adj_index; 3878 int i, err2, err; 3879 3880 if (!nhgi->gateway) 3881 return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3882 3883 for (i = 0; i < nhgi->count; i++) { 3884 nh = &nhgi->nexthops[i]; 3885 3886 if (nh->should_offload != nh->offloaded) { 3887 offload_change = true; 3888 if (nh->should_offload) 3889 nh->update = 1; 3890 } 3891 } 3892 if (!offload_change) { 3893 /* Nothing was added or removed, so no need to reallocate. Just 3894 * update MAC on existing adjacency indexes. 3895 */ 3896 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false); 3897 if (err) { 3898 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3899 goto set_trap; 3900 } 3901 /* Flags of individual nexthop buckets might need to be 3902 * updated. 3903 */ 3904 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3905 return 0; 3906 } 3907 mlxsw_sp_nexthop_group_normalize(nhgi); 3908 if (!nhgi->sum_norm_weight) { 3909 /* No neigh of this group is connected so we just set 3910 * the trap and let everthing flow through kernel. 3911 */ 3912 err = 0; 3913 goto set_trap; 3914 } 3915 3916 ecmp_size = nhgi->sum_norm_weight; 3917 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size); 3918 if (err) 3919 /* No valid allocation size available. */ 3920 goto set_trap; 3921 3922 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3923 ecmp_size, &adj_index); 3924 if (err) { 3925 /* We ran out of KVD linear space, just set the 3926 * trap and let everything flow through kernel. 3927 */ 3928 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n"); 3929 goto set_trap; 3930 } 3931 old_adj_index_valid = nhgi->adj_index_valid; 3932 old_adj_index = nhgi->adj_index; 3933 old_ecmp_size = nhgi->ecmp_size; 3934 nhgi->adj_index_valid = 1; 3935 nhgi->adj_index = adj_index; 3936 nhgi->ecmp_size = ecmp_size; 3937 mlxsw_sp_nexthop_group_rebalance(nhgi); 3938 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true); 3939 if (err) { 3940 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n"); 3941 goto set_trap; 3942 } 3943 3944 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3945 3946 if (!old_adj_index_valid) { 3947 /* The trap was set for fib entries, so we have to call 3948 * fib entry update to unset it and use adjacency index. 3949 */ 3950 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3951 if (err) { 3952 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n"); 3953 goto set_trap; 3954 } 3955 return 0; 3956 } 3957 3958 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp, 3959 old_adj_index, old_ecmp_size); 3960 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3961 old_ecmp_size, old_adj_index); 3962 if (err) { 3963 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n"); 3964 goto set_trap; 3965 } 3966 3967 return 0; 3968 3969 set_trap: 3970 old_adj_index_valid = nhgi->adj_index_valid; 3971 nhgi->adj_index_valid = 0; 3972 for (i = 0; i < nhgi->count; i++) { 3973 nh = &nhgi->nexthops[i]; 3974 nh->offloaded = 0; 3975 } 3976 err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp); 3977 if (err2) 3978 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n"); 3979 mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp); 3980 if (old_adj_index_valid) 3981 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 3982 nhgi->ecmp_size, nhgi->adj_index); 3983 return err; 3984 } 3985 3986 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh, 3987 bool removing) 3988 { 3989 if (!removing) { 3990 nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD; 3991 nh->should_offload = 1; 3992 } else if (nh->nhgi->is_resilient) { 3993 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 3994 nh->should_offload = 1; 3995 } else { 3996 nh->should_offload = 0; 3997 } 3998 nh->update = 1; 3999 } 4000 4001 static int 4002 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp, 4003 struct mlxsw_sp_neigh_entry *neigh_entry) 4004 { 4005 struct neighbour *n, *old_n = neigh_entry->key.n; 4006 struct mlxsw_sp_nexthop *nh; 4007 bool entry_connected; 4008 u8 nud_state, dead; 4009 int err; 4010 4011 nh = list_first_entry(&neigh_entry->nexthop_list, 4012 struct mlxsw_sp_nexthop, neigh_list_node); 4013 4014 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4015 if (!n) { 4016 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4017 if (IS_ERR(n)) 4018 return PTR_ERR(n); 4019 neigh_event_send(n, NULL); 4020 } 4021 4022 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry); 4023 neigh_entry->key.n = n; 4024 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4025 if (err) 4026 goto err_neigh_entry_insert; 4027 4028 read_lock_bh(&n->lock); 4029 nud_state = n->nud_state; 4030 dead = n->dead; 4031 read_unlock_bh(&n->lock); 4032 entry_connected = nud_state & NUD_VALID && !dead; 4033 4034 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4035 neigh_list_node) { 4036 neigh_release(old_n); 4037 neigh_clone(n); 4038 __mlxsw_sp_nexthop_neigh_update(nh, !entry_connected); 4039 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4040 } 4041 4042 neigh_release(n); 4043 4044 return 0; 4045 4046 err_neigh_entry_insert: 4047 neigh_entry->key.n = old_n; 4048 mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry); 4049 neigh_release(n); 4050 return err; 4051 } 4052 4053 static void 4054 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp, 4055 struct mlxsw_sp_neigh_entry *neigh_entry, 4056 bool removing, bool dead) 4057 { 4058 struct mlxsw_sp_nexthop *nh; 4059 4060 if (list_empty(&neigh_entry->nexthop_list)) 4061 return; 4062 4063 if (dead) { 4064 int err; 4065 4066 err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp, 4067 neigh_entry); 4068 if (err) 4069 dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n"); 4070 return; 4071 } 4072 4073 list_for_each_entry(nh, &neigh_entry->nexthop_list, 4074 neigh_list_node) { 4075 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4076 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4077 } 4078 } 4079 4080 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh, 4081 struct mlxsw_sp_rif *rif) 4082 { 4083 if (nh->rif) 4084 return; 4085 4086 nh->rif = rif; 4087 list_add(&nh->rif_list_node, &rif->nexthop_list); 4088 } 4089 4090 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh) 4091 { 4092 if (!nh->rif) 4093 return; 4094 4095 list_del(&nh->rif_list_node); 4096 nh->rif = NULL; 4097 } 4098 4099 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp, 4100 struct mlxsw_sp_nexthop *nh) 4101 { 4102 struct mlxsw_sp_neigh_entry *neigh_entry; 4103 struct neighbour *n; 4104 u8 nud_state, dead; 4105 int err; 4106 4107 if (!nh->nhgi->gateway || nh->neigh_entry) 4108 return 0; 4109 4110 /* Take a reference of neigh here ensuring that neigh would 4111 * not be destructed before the nexthop entry is finished. 4112 * The reference is taken either in neigh_lookup() or 4113 * in neigh_create() in case n is not found. 4114 */ 4115 n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4116 if (!n) { 4117 n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev); 4118 if (IS_ERR(n)) 4119 return PTR_ERR(n); 4120 neigh_event_send(n, NULL); 4121 } 4122 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n); 4123 if (!neigh_entry) { 4124 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n); 4125 if (IS_ERR(neigh_entry)) { 4126 err = -EINVAL; 4127 goto err_neigh_entry_create; 4128 } 4129 } 4130 4131 /* If that is the first nexthop connected to that neigh, add to 4132 * nexthop_neighs_list 4133 */ 4134 if (list_empty(&neigh_entry->nexthop_list)) 4135 list_add_tail(&neigh_entry->nexthop_neighs_list_node, 4136 &mlxsw_sp->router->nexthop_neighs_list); 4137 4138 nh->neigh_entry = neigh_entry; 4139 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list); 4140 read_lock_bh(&n->lock); 4141 nud_state = n->nud_state; 4142 dead = n->dead; 4143 read_unlock_bh(&n->lock); 4144 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead)); 4145 4146 return 0; 4147 4148 err_neigh_entry_create: 4149 neigh_release(n); 4150 return err; 4151 } 4152 4153 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp, 4154 struct mlxsw_sp_nexthop *nh) 4155 { 4156 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry; 4157 struct neighbour *n; 4158 4159 if (!neigh_entry) 4160 return; 4161 n = neigh_entry->key.n; 4162 4163 __mlxsw_sp_nexthop_neigh_update(nh, true); 4164 list_del(&nh->neigh_list_node); 4165 nh->neigh_entry = NULL; 4166 4167 /* If that is the last nexthop connected to that neigh, remove from 4168 * nexthop_neighs_list 4169 */ 4170 if (list_empty(&neigh_entry->nexthop_list)) 4171 list_del(&neigh_entry->nexthop_neighs_list_node); 4172 4173 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list)) 4174 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry); 4175 4176 neigh_release(n); 4177 } 4178 4179 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev) 4180 { 4181 struct net_device *ul_dev; 4182 bool is_up; 4183 4184 rcu_read_lock(); 4185 ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev); 4186 is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true; 4187 rcu_read_unlock(); 4188 4189 return is_up; 4190 } 4191 4192 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp, 4193 struct mlxsw_sp_nexthop *nh, 4194 struct mlxsw_sp_ipip_entry *ipip_entry) 4195 { 4196 bool removing; 4197 4198 if (!nh->nhgi->gateway || nh->ipip_entry) 4199 return; 4200 4201 nh->ipip_entry = ipip_entry; 4202 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev); 4203 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4204 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common); 4205 } 4206 4207 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp, 4208 struct mlxsw_sp_nexthop *nh) 4209 { 4210 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry; 4211 4212 if (!ipip_entry) 4213 return; 4214 4215 __mlxsw_sp_nexthop_neigh_update(nh, true); 4216 nh->ipip_entry = NULL; 4217 } 4218 4219 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp, 4220 const struct fib_nh *fib_nh, 4221 enum mlxsw_sp_ipip_type *p_ipipt) 4222 { 4223 struct net_device *dev = fib_nh->fib_nh_dev; 4224 4225 return dev && 4226 fib_nh->nh_parent->fib_type == RTN_UNICAST && 4227 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt); 4228 } 4229 4230 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp, 4231 struct mlxsw_sp_nexthop *nh, 4232 const struct net_device *dev) 4233 { 4234 const struct mlxsw_sp_ipip_ops *ipip_ops; 4235 struct mlxsw_sp_ipip_entry *ipip_entry; 4236 struct mlxsw_sp_rif *rif; 4237 int err; 4238 4239 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev); 4240 if (ipip_entry) { 4241 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 4242 if (ipip_ops->can_offload(mlxsw_sp, dev)) { 4243 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP; 4244 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry); 4245 return 0; 4246 } 4247 } 4248 4249 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH; 4250 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 4251 if (!rif) 4252 return 0; 4253 4254 mlxsw_sp_nexthop_rif_init(nh, rif); 4255 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh); 4256 if (err) 4257 goto err_neigh_init; 4258 4259 return 0; 4260 4261 err_neigh_init: 4262 mlxsw_sp_nexthop_rif_fini(nh); 4263 return err; 4264 } 4265 4266 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp, 4267 struct mlxsw_sp_nexthop *nh) 4268 { 4269 switch (nh->type) { 4270 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4271 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh); 4272 mlxsw_sp_nexthop_rif_fini(nh); 4273 break; 4274 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4275 mlxsw_sp_nexthop_rif_fini(nh); 4276 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh); 4277 break; 4278 } 4279 } 4280 4281 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp, 4282 struct mlxsw_sp_nexthop_group *nh_grp, 4283 struct mlxsw_sp_nexthop *nh, 4284 struct fib_nh *fib_nh) 4285 { 4286 struct net_device *dev = fib_nh->fib_nh_dev; 4287 struct in_device *in_dev; 4288 int err; 4289 4290 nh->nhgi = nh_grp->nhgi; 4291 nh->key.fib_nh = fib_nh; 4292 #ifdef CONFIG_IP_ROUTE_MULTIPATH 4293 nh->nh_weight = fib_nh->fib_nh_weight; 4294 #else 4295 nh->nh_weight = 1; 4296 #endif 4297 memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4)); 4298 nh->neigh_tbl = &arp_tbl; 4299 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh); 4300 if (err) 4301 return err; 4302 4303 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4304 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4305 4306 if (!dev) 4307 return 0; 4308 nh->ifindex = dev->ifindex; 4309 4310 rcu_read_lock(); 4311 in_dev = __in_dev_get_rcu(dev); 4312 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) && 4313 fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) { 4314 rcu_read_unlock(); 4315 return 0; 4316 } 4317 rcu_read_unlock(); 4318 4319 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4320 if (err) 4321 goto err_nexthop_neigh_init; 4322 4323 return 0; 4324 4325 err_nexthop_neigh_init: 4326 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4327 return err; 4328 } 4329 4330 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp, 4331 struct mlxsw_sp_nexthop *nh) 4332 { 4333 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4334 list_del(&nh->router_list_node); 4335 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4336 mlxsw_sp_nexthop_remove(mlxsw_sp, nh); 4337 } 4338 4339 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp, 4340 unsigned long event, struct fib_nh *fib_nh) 4341 { 4342 struct mlxsw_sp_nexthop_key key; 4343 struct mlxsw_sp_nexthop *nh; 4344 4345 key.fib_nh = fib_nh; 4346 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key); 4347 if (!nh) 4348 return; 4349 4350 switch (event) { 4351 case FIB_EVENT_NH_ADD: 4352 mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev); 4353 break; 4354 case FIB_EVENT_NH_DEL: 4355 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4356 break; 4357 } 4358 4359 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4360 } 4361 4362 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp, 4363 struct mlxsw_sp_rif *rif) 4364 { 4365 struct mlxsw_sp_nexthop *nh; 4366 bool removing; 4367 4368 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) { 4369 switch (nh->type) { 4370 case MLXSW_SP_NEXTHOP_TYPE_ETH: 4371 removing = false; 4372 break; 4373 case MLXSW_SP_NEXTHOP_TYPE_IPIP: 4374 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev); 4375 break; 4376 default: 4377 WARN_ON(1); 4378 continue; 4379 } 4380 4381 __mlxsw_sp_nexthop_neigh_update(nh, removing); 4382 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4383 } 4384 } 4385 4386 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp, 4387 struct mlxsw_sp_rif *old_rif, 4388 struct mlxsw_sp_rif *new_rif) 4389 { 4390 struct mlxsw_sp_nexthop *nh; 4391 4392 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list); 4393 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node) 4394 nh->rif = new_rif; 4395 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif); 4396 } 4397 4398 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 4399 struct mlxsw_sp_rif *rif) 4400 { 4401 struct mlxsw_sp_nexthop *nh, *tmp; 4402 4403 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) { 4404 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4405 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp); 4406 } 4407 } 4408 4409 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp) 4410 { 4411 enum mlxsw_reg_ratr_trap_action trap_action; 4412 char ratr_pl[MLXSW_REG_RATR_LEN]; 4413 int err; 4414 4415 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4416 &mlxsw_sp->router->adj_trap_index); 4417 if (err) 4418 return err; 4419 4420 trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP; 4421 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true, 4422 MLXSW_REG_RATR_TYPE_ETHERNET, 4423 mlxsw_sp->router->adj_trap_index, 4424 mlxsw_sp->router->lb_rif_index); 4425 mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action); 4426 mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0); 4427 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 4428 if (err) 4429 goto err_ratr_write; 4430 4431 return 0; 4432 4433 err_ratr_write: 4434 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4435 mlxsw_sp->router->adj_trap_index); 4436 return err; 4437 } 4438 4439 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp) 4440 { 4441 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 4442 mlxsw_sp->router->adj_trap_index); 4443 } 4444 4445 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp) 4446 { 4447 int err; 4448 4449 if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups)) 4450 return 0; 4451 4452 err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp); 4453 if (err) 4454 return err; 4455 4456 refcount_set(&mlxsw_sp->router->num_groups, 1); 4457 4458 return 0; 4459 } 4460 4461 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp) 4462 { 4463 if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups)) 4464 return; 4465 4466 mlxsw_sp_adj_trap_entry_fini(mlxsw_sp); 4467 } 4468 4469 static void 4470 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp, 4471 const struct mlxsw_sp_nexthop_group *nh_grp, 4472 unsigned long *activity) 4473 { 4474 char *ratrad_pl; 4475 int i, err; 4476 4477 ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL); 4478 if (!ratrad_pl) 4479 return; 4480 4481 mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index, 4482 nh_grp->nhgi->count); 4483 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl); 4484 if (err) 4485 goto out; 4486 4487 for (i = 0; i < nh_grp->nhgi->count; i++) { 4488 if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i)) 4489 continue; 4490 bitmap_set(activity, i, 1); 4491 } 4492 4493 out: 4494 kfree(ratrad_pl); 4495 } 4496 4497 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */ 4498 4499 static void 4500 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp, 4501 const struct mlxsw_sp_nexthop_group *nh_grp) 4502 { 4503 unsigned long *activity; 4504 4505 activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL); 4506 if (!activity) 4507 return; 4508 4509 mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity); 4510 nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id, 4511 nh_grp->nhgi->count, activity); 4512 4513 bitmap_free(activity); 4514 } 4515 4516 static void 4517 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp) 4518 { 4519 unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL; 4520 4521 mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw, 4522 msecs_to_jiffies(interval)); 4523 } 4524 4525 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work) 4526 { 4527 struct mlxsw_sp_nexthop_group_info *nhgi; 4528 struct mlxsw_sp_router *router; 4529 bool reschedule = false; 4530 4531 router = container_of(work, struct mlxsw_sp_router, 4532 nh_grp_activity_dw.work); 4533 4534 mutex_lock(&router->lock); 4535 4536 list_for_each_entry(nhgi, &router->nh_res_grp_list, list) { 4537 mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp); 4538 reschedule = true; 4539 } 4540 4541 mutex_unlock(&router->lock); 4542 4543 if (!reschedule) 4544 return; 4545 mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp); 4546 } 4547 4548 static int 4549 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp, 4550 const struct nh_notifier_single_info *nh, 4551 struct netlink_ext_ack *extack) 4552 { 4553 int err = -EINVAL; 4554 4555 if (nh->is_fdb) 4556 NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported"); 4557 else if (nh->has_encap) 4558 NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported"); 4559 else 4560 err = 0; 4561 4562 return err; 4563 } 4564 4565 static int 4566 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp, 4567 const struct nh_notifier_single_info *nh, 4568 struct netlink_ext_ack *extack) 4569 { 4570 int err; 4571 4572 err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack); 4573 if (err) 4574 return err; 4575 4576 /* Device only nexthops with an IPIP device are programmed as 4577 * encapsulating adjacency entries. 4578 */ 4579 if (!nh->gw_family && !nh->is_reject && 4580 !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) { 4581 NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway"); 4582 return -EINVAL; 4583 } 4584 4585 return 0; 4586 } 4587 4588 static int 4589 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp, 4590 const struct nh_notifier_grp_info *nh_grp, 4591 struct netlink_ext_ack *extack) 4592 { 4593 int i; 4594 4595 if (nh_grp->is_fdb) { 4596 NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported"); 4597 return -EINVAL; 4598 } 4599 4600 for (i = 0; i < nh_grp->num_nh; i++) { 4601 const struct nh_notifier_single_info *nh; 4602 int err; 4603 4604 nh = &nh_grp->nh_entries[i].nh; 4605 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4606 extack); 4607 if (err) 4608 return err; 4609 } 4610 4611 return 0; 4612 } 4613 4614 static int 4615 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp, 4616 const struct nh_notifier_res_table_info *nh_res_table, 4617 struct netlink_ext_ack *extack) 4618 { 4619 unsigned int alloc_size; 4620 bool valid_size = false; 4621 int err, i; 4622 4623 if (nh_res_table->num_nh_buckets < 32) { 4624 NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32"); 4625 return -EINVAL; 4626 } 4627 4628 for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) { 4629 const struct mlxsw_sp_adj_grp_size_range *size_range; 4630 4631 size_range = &mlxsw_sp->router->adj_grp_size_ranges[i]; 4632 4633 if (nh_res_table->num_nh_buckets >= size_range->start && 4634 nh_res_table->num_nh_buckets <= size_range->end) { 4635 valid_size = true; 4636 break; 4637 } 4638 } 4639 4640 if (!valid_size) { 4641 NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets"); 4642 return -EINVAL; 4643 } 4644 4645 err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp, 4646 MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 4647 nh_res_table->num_nh_buckets, 4648 &alloc_size); 4649 if (err || nh_res_table->num_nh_buckets != alloc_size) { 4650 NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition"); 4651 return -EINVAL; 4652 } 4653 4654 return 0; 4655 } 4656 4657 static int 4658 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp, 4659 const struct nh_notifier_res_table_info *nh_res_table, 4660 struct netlink_ext_ack *extack) 4661 { 4662 int err; 4663 u16 i; 4664 4665 err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp, 4666 nh_res_table, 4667 extack); 4668 if (err) 4669 return err; 4670 4671 for (i = 0; i < nh_res_table->num_nh_buckets; i++) { 4672 const struct nh_notifier_single_info *nh; 4673 int err; 4674 4675 nh = &nh_res_table->nhs[i]; 4676 err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4677 extack); 4678 if (err) 4679 return err; 4680 } 4681 4682 return 0; 4683 } 4684 4685 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp, 4686 unsigned long event, 4687 struct nh_notifier_info *info) 4688 { 4689 struct nh_notifier_single_info *nh; 4690 4691 if (event != NEXTHOP_EVENT_REPLACE && 4692 event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE && 4693 event != NEXTHOP_EVENT_BUCKET_REPLACE) 4694 return 0; 4695 4696 switch (info->type) { 4697 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4698 return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh, 4699 info->extack); 4700 case NH_NOTIFIER_INFO_TYPE_GRP: 4701 return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp, 4702 info->nh_grp, 4703 info->extack); 4704 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4705 return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp, 4706 info->nh_res_table, 4707 info->extack); 4708 case NH_NOTIFIER_INFO_TYPE_RES_BUCKET: 4709 nh = &info->nh_res_bucket->new_nh; 4710 return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh, 4711 info->extack); 4712 default: 4713 NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type"); 4714 return -EOPNOTSUPP; 4715 } 4716 } 4717 4718 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp, 4719 const struct nh_notifier_info *info) 4720 { 4721 const struct net_device *dev; 4722 4723 switch (info->type) { 4724 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4725 dev = info->nh->dev; 4726 return info->nh->gw_family || info->nh->is_reject || 4727 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL); 4728 case NH_NOTIFIER_INFO_TYPE_GRP: 4729 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4730 /* Already validated earlier. */ 4731 return true; 4732 default: 4733 return false; 4734 } 4735 } 4736 4737 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp, 4738 struct mlxsw_sp_nexthop *nh) 4739 { 4740 u16 lb_rif_index = mlxsw_sp->router->lb_rif_index; 4741 4742 nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD; 4743 nh->should_offload = 1; 4744 /* While nexthops that discard packets do not forward packets 4745 * via an egress RIF, they still need to be programmed using a 4746 * valid RIF, so use the loopback RIF created during init. 4747 */ 4748 nh->rif = mlxsw_sp->router->rifs[lb_rif_index]; 4749 } 4750 4751 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp, 4752 struct mlxsw_sp_nexthop *nh) 4753 { 4754 nh->rif = NULL; 4755 nh->should_offload = 0; 4756 } 4757 4758 static int 4759 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp, 4760 struct mlxsw_sp_nexthop_group *nh_grp, 4761 struct mlxsw_sp_nexthop *nh, 4762 struct nh_notifier_single_info *nh_obj, int weight) 4763 { 4764 struct net_device *dev = nh_obj->dev; 4765 int err; 4766 4767 nh->nhgi = nh_grp->nhgi; 4768 nh->nh_weight = weight; 4769 4770 switch (nh_obj->gw_family) { 4771 case AF_INET: 4772 memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4)); 4773 nh->neigh_tbl = &arp_tbl; 4774 break; 4775 case AF_INET6: 4776 memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6)); 4777 #if IS_ENABLED(CONFIG_IPV6) 4778 nh->neigh_tbl = &nd_tbl; 4779 #endif 4780 break; 4781 } 4782 4783 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 4784 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 4785 nh->ifindex = dev->ifindex; 4786 4787 err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 4788 if (err) 4789 goto err_type_init; 4790 4791 if (nh_obj->is_reject) 4792 mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh); 4793 4794 /* In a resilient nexthop group, all the nexthops must be written to 4795 * the adjacency table. Even if they do not have a valid neighbour or 4796 * RIF. 4797 */ 4798 if (nh_grp->nhgi->is_resilient && !nh->should_offload) { 4799 nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP; 4800 nh->should_offload = 1; 4801 } 4802 4803 return 0; 4804 4805 err_type_init: 4806 list_del(&nh->router_list_node); 4807 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4808 return err; 4809 } 4810 4811 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp, 4812 struct mlxsw_sp_nexthop *nh) 4813 { 4814 if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD) 4815 mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh); 4816 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 4817 list_del(&nh->router_list_node); 4818 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 4819 nh->should_offload = 0; 4820 } 4821 4822 static int 4823 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp, 4824 struct mlxsw_sp_nexthop_group *nh_grp, 4825 struct nh_notifier_info *info) 4826 { 4827 struct mlxsw_sp_nexthop_group_info *nhgi; 4828 struct mlxsw_sp_nexthop *nh; 4829 bool is_resilient = false; 4830 unsigned int nhs; 4831 int err, i; 4832 4833 switch (info->type) { 4834 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4835 nhs = 1; 4836 break; 4837 case NH_NOTIFIER_INFO_TYPE_GRP: 4838 nhs = info->nh_grp->num_nh; 4839 break; 4840 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4841 nhs = info->nh_res_table->num_nh_buckets; 4842 is_resilient = true; 4843 break; 4844 default: 4845 return -EINVAL; 4846 } 4847 4848 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 4849 if (!nhgi) 4850 return -ENOMEM; 4851 nh_grp->nhgi = nhgi; 4852 nhgi->nh_grp = nh_grp; 4853 nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info); 4854 nhgi->is_resilient = is_resilient; 4855 nhgi->count = nhs; 4856 for (i = 0; i < nhgi->count; i++) { 4857 struct nh_notifier_single_info *nh_obj; 4858 int weight; 4859 4860 nh = &nhgi->nexthops[i]; 4861 switch (info->type) { 4862 case NH_NOTIFIER_INFO_TYPE_SINGLE: 4863 nh_obj = info->nh; 4864 weight = 1; 4865 break; 4866 case NH_NOTIFIER_INFO_TYPE_GRP: 4867 nh_obj = &info->nh_grp->nh_entries[i].nh; 4868 weight = info->nh_grp->nh_entries[i].weight; 4869 break; 4870 case NH_NOTIFIER_INFO_TYPE_RES_TABLE: 4871 nh_obj = &info->nh_res_table->nhs[i]; 4872 weight = 1; 4873 break; 4874 default: 4875 err = -EINVAL; 4876 goto err_nexthop_obj_init; 4877 } 4878 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 4879 weight); 4880 if (err) 4881 goto err_nexthop_obj_init; 4882 } 4883 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 4884 if (err) 4885 goto err_group_inc; 4886 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4887 if (err) { 4888 NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device"); 4889 goto err_group_refresh; 4890 } 4891 4892 /* Add resilient nexthop groups to a list so that the activity of their 4893 * nexthop buckets will be periodically queried and cleared. 4894 */ 4895 if (nhgi->is_resilient) { 4896 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4897 mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp); 4898 list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list); 4899 } 4900 4901 return 0; 4902 4903 err_group_refresh: 4904 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 4905 err_group_inc: 4906 i = nhgi->count; 4907 err_nexthop_obj_init: 4908 for (i--; i >= 0; i--) { 4909 nh = &nhgi->nexthops[i]; 4910 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4911 } 4912 kfree(nhgi); 4913 return err; 4914 } 4915 4916 static void 4917 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp, 4918 struct mlxsw_sp_nexthop_group *nh_grp) 4919 { 4920 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 4921 struct mlxsw_sp_router *router = mlxsw_sp->router; 4922 int i; 4923 4924 if (nhgi->is_resilient) { 4925 list_del(&nhgi->list); 4926 if (list_empty(&mlxsw_sp->router->nh_res_grp_list)) 4927 cancel_delayed_work(&router->nh_grp_activity_dw); 4928 } 4929 4930 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 4931 for (i = nhgi->count - 1; i >= 0; i--) { 4932 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 4933 4934 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 4935 } 4936 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 4937 WARN_ON_ONCE(nhgi->adj_index_valid); 4938 kfree(nhgi); 4939 } 4940 4941 static struct mlxsw_sp_nexthop_group * 4942 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp, 4943 struct nh_notifier_info *info) 4944 { 4945 struct mlxsw_sp_nexthop_group *nh_grp; 4946 int err; 4947 4948 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 4949 if (!nh_grp) 4950 return ERR_PTR(-ENOMEM); 4951 INIT_LIST_HEAD(&nh_grp->vr_list); 4952 err = rhashtable_init(&nh_grp->vr_ht, 4953 &mlxsw_sp_nexthop_group_vr_ht_params); 4954 if (err) 4955 goto err_nexthop_group_vr_ht_init; 4956 INIT_LIST_HEAD(&nh_grp->fib_list); 4957 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4958 nh_grp->obj.id = info->id; 4959 4960 err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info); 4961 if (err) 4962 goto err_nexthop_group_info_init; 4963 4964 nh_grp->can_destroy = false; 4965 4966 return nh_grp; 4967 4968 err_nexthop_group_info_init: 4969 rhashtable_destroy(&nh_grp->vr_ht); 4970 err_nexthop_group_vr_ht_init: 4971 kfree(nh_grp); 4972 return ERR_PTR(err); 4973 } 4974 4975 static void 4976 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp, 4977 struct mlxsw_sp_nexthop_group *nh_grp) 4978 { 4979 if (!nh_grp->can_destroy) 4980 return; 4981 mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp); 4982 WARN_ON_ONCE(!list_empty(&nh_grp->fib_list)); 4983 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 4984 rhashtable_destroy(&nh_grp->vr_ht); 4985 kfree(nh_grp); 4986 } 4987 4988 static struct mlxsw_sp_nexthop_group * 4989 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id) 4990 { 4991 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg; 4992 4993 cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ; 4994 cmp_arg.id = id; 4995 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, 4996 &cmp_arg, 4997 mlxsw_sp_nexthop_group_ht_params); 4998 } 4999 5000 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp, 5001 struct mlxsw_sp_nexthop_group *nh_grp) 5002 { 5003 return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5004 } 5005 5006 static int 5007 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp, 5008 struct mlxsw_sp_nexthop_group *nh_grp, 5009 struct mlxsw_sp_nexthop_group *old_nh_grp, 5010 struct netlink_ext_ack *extack) 5011 { 5012 struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi; 5013 struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi; 5014 int err; 5015 5016 old_nh_grp->nhgi = new_nhgi; 5017 new_nhgi->nh_grp = old_nh_grp; 5018 nh_grp->nhgi = old_nhgi; 5019 old_nhgi->nh_grp = nh_grp; 5020 5021 if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5022 /* Both the old adjacency index and the new one are valid. 5023 * Routes are currently using the old one. Tell the device to 5024 * replace the old adjacency index with the new one. 5025 */ 5026 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp, 5027 old_nhgi->adj_index, 5028 old_nhgi->ecmp_size); 5029 if (err) { 5030 NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one"); 5031 goto err_out; 5032 } 5033 } else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) { 5034 /* The old adjacency index is valid, while the new one is not. 5035 * Iterate over all the routes using the group and change them 5036 * to trap packets to the CPU. 5037 */ 5038 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5039 if (err) { 5040 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets"); 5041 goto err_out; 5042 } 5043 } else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) { 5044 /* The old adjacency index is invalid, while the new one is. 5045 * Iterate over all the routes using the group and change them 5046 * to forward packets using the new valid index. 5047 */ 5048 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp); 5049 if (err) { 5050 NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets"); 5051 goto err_out; 5052 } 5053 } 5054 5055 /* Make sure the flags are set / cleared based on the new nexthop group 5056 * information. 5057 */ 5058 mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp); 5059 5060 /* At this point 'nh_grp' is just a shell that is not used by anyone 5061 * and its nexthop group info is the old info that was just replaced 5062 * with the new one. Remove it. 5063 */ 5064 nh_grp->can_destroy = true; 5065 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5066 5067 return 0; 5068 5069 err_out: 5070 old_nhgi->nh_grp = old_nh_grp; 5071 nh_grp->nhgi = new_nhgi; 5072 new_nhgi->nh_grp = nh_grp; 5073 old_nh_grp->nhgi = old_nhgi; 5074 return err; 5075 } 5076 5077 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp, 5078 struct nh_notifier_info *info) 5079 { 5080 struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp; 5081 struct netlink_ext_ack *extack = info->extack; 5082 int err; 5083 5084 nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info); 5085 if (IS_ERR(nh_grp)) 5086 return PTR_ERR(nh_grp); 5087 5088 old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5089 if (!old_nh_grp) 5090 err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp); 5091 else 5092 err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp, 5093 old_nh_grp, extack); 5094 5095 if (err) { 5096 nh_grp->can_destroy = true; 5097 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5098 } 5099 5100 return err; 5101 } 5102 5103 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp, 5104 struct nh_notifier_info *info) 5105 { 5106 struct mlxsw_sp_nexthop_group *nh_grp; 5107 5108 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5109 if (!nh_grp) 5110 return; 5111 5112 nh_grp->can_destroy = true; 5113 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5114 5115 /* If the group still has routes using it, then defer the delete 5116 * operation until the last route using it is deleted. 5117 */ 5118 if (!list_empty(&nh_grp->fib_list)) 5119 return; 5120 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5121 } 5122 5123 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp, 5124 u32 adj_index, char *ratr_pl) 5125 { 5126 MLXSW_REG_ZERO(ratr, ratr_pl); 5127 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5128 mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index); 5129 mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16); 5130 5131 return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl); 5132 } 5133 5134 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new) 5135 { 5136 /* Clear the opcode and activity on both the old and new payload as 5137 * they are irrelevant for the comparison. 5138 */ 5139 mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ); 5140 mlxsw_reg_ratr_a_set(ratr_pl, 0); 5141 mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ); 5142 mlxsw_reg_ratr_a_set(ratr_pl_new, 0); 5143 5144 /* If the contents of the adjacency entry are consistent with the 5145 * replacement request, then replacement was successful. 5146 */ 5147 if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN)) 5148 return 0; 5149 5150 return -EINVAL; 5151 } 5152 5153 static int 5154 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp, 5155 struct mlxsw_sp_nexthop *nh, 5156 struct nh_notifier_info *info) 5157 { 5158 u16 bucket_index = info->nh_res_bucket->bucket_index; 5159 struct netlink_ext_ack *extack = info->extack; 5160 bool force = info->nh_res_bucket->force; 5161 char ratr_pl_new[MLXSW_REG_RATR_LEN]; 5162 char ratr_pl[MLXSW_REG_RATR_LEN]; 5163 u32 adj_index; 5164 int err; 5165 5166 /* No point in trying an atomic replacement if the idle timer interval 5167 * is smaller than the interval in which we query and clear activity. 5168 */ 5169 if (!force && info->nh_res_bucket->idle_timer_ms < 5170 MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL) 5171 force = true; 5172 5173 adj_index = nh->nhgi->adj_index + bucket_index; 5174 err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl); 5175 if (err) { 5176 NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket"); 5177 return err; 5178 } 5179 5180 if (!force) { 5181 err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index, 5182 ratr_pl_new); 5183 if (err) { 5184 NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent"); 5185 return err; 5186 } 5187 5188 err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new); 5189 if (err) { 5190 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement"); 5191 return err; 5192 } 5193 } 5194 5195 nh->update = 0; 5196 nh->offloaded = 1; 5197 mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index); 5198 5199 return 0; 5200 } 5201 5202 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp, 5203 struct nh_notifier_info *info) 5204 { 5205 u16 bucket_index = info->nh_res_bucket->bucket_index; 5206 struct netlink_ext_ack *extack = info->extack; 5207 struct mlxsw_sp_nexthop_group_info *nhgi; 5208 struct nh_notifier_single_info *nh_obj; 5209 struct mlxsw_sp_nexthop_group *nh_grp; 5210 struct mlxsw_sp_nexthop *nh; 5211 int err; 5212 5213 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id); 5214 if (!nh_grp) { 5215 NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found"); 5216 return -EINVAL; 5217 } 5218 5219 nhgi = nh_grp->nhgi; 5220 5221 if (bucket_index >= nhgi->count) { 5222 NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range"); 5223 return -EINVAL; 5224 } 5225 5226 nh = &nhgi->nexthops[bucket_index]; 5227 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5228 5229 nh_obj = &info->nh_res_bucket->new_nh; 5230 err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5231 if (err) { 5232 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement"); 5233 goto err_nexthop_obj_init; 5234 } 5235 5236 err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info); 5237 if (err) 5238 goto err_nexthop_obj_bucket_adj_update; 5239 5240 return 0; 5241 5242 err_nexthop_obj_bucket_adj_update: 5243 mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh); 5244 err_nexthop_obj_init: 5245 nh_obj = &info->nh_res_bucket->old_nh; 5246 mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1); 5247 /* The old adjacency entry was not overwritten */ 5248 nh->update = 0; 5249 nh->offloaded = 1; 5250 return err; 5251 } 5252 5253 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb, 5254 unsigned long event, void *ptr) 5255 { 5256 struct nh_notifier_info *info = ptr; 5257 struct mlxsw_sp_router *router; 5258 int err = 0; 5259 5260 router = container_of(nb, struct mlxsw_sp_router, nexthop_nb); 5261 err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info); 5262 if (err) 5263 goto out; 5264 5265 mutex_lock(&router->lock); 5266 5267 switch (event) { 5268 case NEXTHOP_EVENT_REPLACE: 5269 err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info); 5270 break; 5271 case NEXTHOP_EVENT_DEL: 5272 mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info); 5273 break; 5274 case NEXTHOP_EVENT_BUCKET_REPLACE: 5275 err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp, 5276 info); 5277 break; 5278 default: 5279 break; 5280 } 5281 5282 mutex_unlock(&router->lock); 5283 5284 out: 5285 return notifier_from_errno(err); 5286 } 5287 5288 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp, 5289 struct fib_info *fi) 5290 { 5291 const struct fib_nh *nh = fib_info_nh(fi, 0); 5292 5293 return nh->fib_nh_scope == RT_SCOPE_LINK || 5294 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL); 5295 } 5296 5297 static int 5298 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp, 5299 struct mlxsw_sp_nexthop_group *nh_grp) 5300 { 5301 unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi); 5302 struct mlxsw_sp_nexthop_group_info *nhgi; 5303 struct mlxsw_sp_nexthop *nh; 5304 int err, i; 5305 5306 nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL); 5307 if (!nhgi) 5308 return -ENOMEM; 5309 nh_grp->nhgi = nhgi; 5310 nhgi->nh_grp = nh_grp; 5311 nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi); 5312 nhgi->count = nhs; 5313 for (i = 0; i < nhgi->count; i++) { 5314 struct fib_nh *fib_nh; 5315 5316 nh = &nhgi->nexthops[i]; 5317 fib_nh = fib_info_nh(nh_grp->ipv4.fi, i); 5318 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh); 5319 if (err) 5320 goto err_nexthop4_init; 5321 } 5322 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 5323 if (err) 5324 goto err_group_inc; 5325 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5326 if (err) 5327 goto err_group_refresh; 5328 5329 return 0; 5330 5331 err_group_refresh: 5332 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5333 err_group_inc: 5334 i = nhgi->count; 5335 err_nexthop4_init: 5336 for (i--; i >= 0; i--) { 5337 nh = &nhgi->nexthops[i]; 5338 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5339 } 5340 kfree(nhgi); 5341 return err; 5342 } 5343 5344 static void 5345 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp, 5346 struct mlxsw_sp_nexthop_group *nh_grp) 5347 { 5348 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 5349 int i; 5350 5351 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 5352 for (i = nhgi->count - 1; i >= 0; i--) { 5353 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 5354 5355 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh); 5356 } 5357 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 5358 WARN_ON_ONCE(nhgi->adj_index_valid); 5359 kfree(nhgi); 5360 } 5361 5362 static struct mlxsw_sp_nexthop_group * 5363 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi) 5364 { 5365 struct mlxsw_sp_nexthop_group *nh_grp; 5366 int err; 5367 5368 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 5369 if (!nh_grp) 5370 return ERR_PTR(-ENOMEM); 5371 INIT_LIST_HEAD(&nh_grp->vr_list); 5372 err = rhashtable_init(&nh_grp->vr_ht, 5373 &mlxsw_sp_nexthop_group_vr_ht_params); 5374 if (err) 5375 goto err_nexthop_group_vr_ht_init; 5376 INIT_LIST_HEAD(&nh_grp->fib_list); 5377 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4; 5378 nh_grp->ipv4.fi = fi; 5379 fib_info_hold(fi); 5380 5381 err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp); 5382 if (err) 5383 goto err_nexthop_group_info_init; 5384 5385 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 5386 if (err) 5387 goto err_nexthop_group_insert; 5388 5389 nh_grp->can_destroy = true; 5390 5391 return nh_grp; 5392 5393 err_nexthop_group_insert: 5394 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5395 err_nexthop_group_info_init: 5396 fib_info_put(fi); 5397 rhashtable_destroy(&nh_grp->vr_ht); 5398 err_nexthop_group_vr_ht_init: 5399 kfree(nh_grp); 5400 return ERR_PTR(err); 5401 } 5402 5403 static void 5404 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp, 5405 struct mlxsw_sp_nexthop_group *nh_grp) 5406 { 5407 if (!nh_grp->can_destroy) 5408 return; 5409 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 5410 mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp); 5411 fib_info_put(nh_grp->ipv4.fi); 5412 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 5413 rhashtable_destroy(&nh_grp->vr_ht); 5414 kfree(nh_grp); 5415 } 5416 5417 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp, 5418 struct mlxsw_sp_fib_entry *fib_entry, 5419 struct fib_info *fi) 5420 { 5421 struct mlxsw_sp_nexthop_group *nh_grp; 5422 5423 if (fi->nh) { 5424 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 5425 fi->nh->id); 5426 if (WARN_ON_ONCE(!nh_grp)) 5427 return -EINVAL; 5428 goto out; 5429 } 5430 5431 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi); 5432 if (!nh_grp) { 5433 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi); 5434 if (IS_ERR(nh_grp)) 5435 return PTR_ERR(nh_grp); 5436 } 5437 out: 5438 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list); 5439 fib_entry->nh_group = nh_grp; 5440 return 0; 5441 } 5442 5443 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp, 5444 struct mlxsw_sp_fib_entry *fib_entry) 5445 { 5446 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 5447 5448 list_del(&fib_entry->nexthop_group_node); 5449 if (!list_empty(&nh_grp->fib_list)) 5450 return; 5451 5452 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 5453 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 5454 return; 5455 } 5456 5457 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp); 5458 } 5459 5460 static bool 5461 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5462 { 5463 struct mlxsw_sp_fib4_entry *fib4_entry; 5464 5465 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5466 common); 5467 return !fib4_entry->dscp; 5468 } 5469 5470 static bool 5471 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry) 5472 { 5473 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5474 5475 switch (fib_entry->fib_node->fib->proto) { 5476 case MLXSW_SP_L3_PROTO_IPV4: 5477 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry)) 5478 return false; 5479 break; 5480 case MLXSW_SP_L3_PROTO_IPV6: 5481 break; 5482 } 5483 5484 switch (fib_entry->type) { 5485 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5486 return !!nh_group->nhgi->adj_index_valid; 5487 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5488 return !!nh_group->nhgi->nh_rif; 5489 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5490 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5491 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5492 return true; 5493 default: 5494 return false; 5495 } 5496 } 5497 5498 static struct mlxsw_sp_nexthop * 5499 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp, 5500 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 5501 { 5502 int i; 5503 5504 for (i = 0; i < nh_grp->nhgi->count; i++) { 5505 struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i]; 5506 struct fib6_info *rt = mlxsw_sp_rt6->rt; 5507 5508 if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev && 5509 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr, 5510 &rt->fib6_nh->fib_nh_gw6)) 5511 return nh; 5512 } 5513 5514 return NULL; 5515 } 5516 5517 static void 5518 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5519 struct fib_entry_notifier_info *fen_info) 5520 { 5521 u32 *p_dst = (u32 *) &fen_info->dst; 5522 struct fib_rt_info fri; 5523 5524 fri.fi = fen_info->fi; 5525 fri.tb_id = fen_info->tb_id; 5526 fri.dst = cpu_to_be32(*p_dst); 5527 fri.dst_len = fen_info->dst_len; 5528 fri.dscp = fen_info->dscp; 5529 fri.type = fen_info->type; 5530 fri.offload = false; 5531 fri.trap = false; 5532 fri.offload_failed = true; 5533 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5534 } 5535 5536 static void 5537 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5538 struct mlxsw_sp_fib_entry *fib_entry) 5539 { 5540 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5541 int dst_len = fib_entry->fib_node->key.prefix_len; 5542 struct mlxsw_sp_fib4_entry *fib4_entry; 5543 struct fib_rt_info fri; 5544 bool should_offload; 5545 5546 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5547 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5548 common); 5549 fri.fi = fib4_entry->fi; 5550 fri.tb_id = fib4_entry->tb_id; 5551 fri.dst = cpu_to_be32(*p_dst); 5552 fri.dst_len = dst_len; 5553 fri.dscp = fib4_entry->dscp; 5554 fri.type = fib4_entry->type; 5555 fri.offload = should_offload; 5556 fri.trap = !should_offload; 5557 fri.offload_failed = false; 5558 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5559 } 5560 5561 static void 5562 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5563 struct mlxsw_sp_fib_entry *fib_entry) 5564 { 5565 u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr; 5566 int dst_len = fib_entry->fib_node->key.prefix_len; 5567 struct mlxsw_sp_fib4_entry *fib4_entry; 5568 struct fib_rt_info fri; 5569 5570 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry, 5571 common); 5572 fri.fi = fib4_entry->fi; 5573 fri.tb_id = fib4_entry->tb_id; 5574 fri.dst = cpu_to_be32(*p_dst); 5575 fri.dst_len = dst_len; 5576 fri.dscp = fib4_entry->dscp; 5577 fri.type = fib4_entry->type; 5578 fri.offload = false; 5579 fri.trap = false; 5580 fri.offload_failed = false; 5581 fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri); 5582 } 5583 5584 #if IS_ENABLED(CONFIG_IPV6) 5585 static void 5586 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5587 struct fib6_info **rt_arr, 5588 unsigned int nrt6) 5589 { 5590 int i; 5591 5592 /* In IPv6 a multipath route is represented using multiple routes, so 5593 * we need to set the flags on all of them. 5594 */ 5595 for (i = 0; i < nrt6; i++) 5596 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i], 5597 false, false, true); 5598 } 5599 #else 5600 static void 5601 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp, 5602 struct fib6_info **rt_arr, 5603 unsigned int nrt6) 5604 { 5605 } 5606 #endif 5607 5608 #if IS_ENABLED(CONFIG_IPV6) 5609 static void 5610 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5611 struct mlxsw_sp_fib_entry *fib_entry) 5612 { 5613 struct mlxsw_sp_fib6_entry *fib6_entry; 5614 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5615 bool should_offload; 5616 5617 should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry); 5618 5619 /* In IPv6 a multipath route is represented using multiple routes, so 5620 * we need to set the flags on all of them. 5621 */ 5622 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5623 common); 5624 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5625 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5626 should_offload, !should_offload, false); 5627 } 5628 #else 5629 static void 5630 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5631 struct mlxsw_sp_fib_entry *fib_entry) 5632 { 5633 } 5634 #endif 5635 5636 #if IS_ENABLED(CONFIG_IPV6) 5637 static void 5638 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5639 struct mlxsw_sp_fib_entry *fib_entry) 5640 { 5641 struct mlxsw_sp_fib6_entry *fib6_entry; 5642 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 5643 5644 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry, 5645 common); 5646 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) 5647 fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt, 5648 false, false, false); 5649 } 5650 #else 5651 static void 5652 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5653 struct mlxsw_sp_fib_entry *fib_entry) 5654 { 5655 } 5656 #endif 5657 5658 static void 5659 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp, 5660 struct mlxsw_sp_fib_entry *fib_entry) 5661 { 5662 switch (fib_entry->fib_node->fib->proto) { 5663 case MLXSW_SP_L3_PROTO_IPV4: 5664 mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry); 5665 break; 5666 case MLXSW_SP_L3_PROTO_IPV6: 5667 mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry); 5668 break; 5669 } 5670 } 5671 5672 static void 5673 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp, 5674 struct mlxsw_sp_fib_entry *fib_entry) 5675 { 5676 switch (fib_entry->fib_node->fib->proto) { 5677 case MLXSW_SP_L3_PROTO_IPV4: 5678 mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5679 break; 5680 case MLXSW_SP_L3_PROTO_IPV6: 5681 mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5682 break; 5683 } 5684 } 5685 5686 static void 5687 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp, 5688 struct mlxsw_sp_fib_entry *fib_entry, 5689 enum mlxsw_reg_ralue_op op) 5690 { 5691 switch (op) { 5692 case MLXSW_REG_RALUE_OP_WRITE_WRITE: 5693 mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry); 5694 break; 5695 case MLXSW_REG_RALUE_OP_WRITE_DELETE: 5696 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry); 5697 break; 5698 default: 5699 break; 5700 } 5701 } 5702 5703 static void 5704 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl, 5705 const struct mlxsw_sp_fib_entry *fib_entry, 5706 enum mlxsw_reg_ralue_op op) 5707 { 5708 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib; 5709 enum mlxsw_reg_ralxx_protocol proto; 5710 u32 *p_dip; 5711 5712 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto; 5713 5714 switch (fib->proto) { 5715 case MLXSW_SP_L3_PROTO_IPV4: 5716 p_dip = (u32 *) fib_entry->fib_node->key.addr; 5717 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id, 5718 fib_entry->fib_node->key.prefix_len, 5719 *p_dip); 5720 break; 5721 case MLXSW_SP_L3_PROTO_IPV6: 5722 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id, 5723 fib_entry->fib_node->key.prefix_len, 5724 fib_entry->fib_node->key.addr); 5725 break; 5726 } 5727 } 5728 5729 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp, 5730 struct mlxsw_sp_fib_entry *fib_entry, 5731 enum mlxsw_reg_ralue_op op) 5732 { 5733 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group; 5734 struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi; 5735 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5736 enum mlxsw_reg_ralue_trap_action trap_action; 5737 u16 trap_id = 0; 5738 u32 adjacency_index = 0; 5739 u16 ecmp_size = 0; 5740 5741 /* In case the nexthop group adjacency index is valid, use it 5742 * with provided ECMP size. Otherwise, setup trap and pass 5743 * traffic to kernel. 5744 */ 5745 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5746 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5747 adjacency_index = nhgi->adj_index; 5748 ecmp_size = nhgi->ecmp_size; 5749 } else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) { 5750 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5751 adjacency_index = mlxsw_sp->router->adj_trap_index; 5752 ecmp_size = 1; 5753 } else { 5754 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5755 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5756 } 5757 5758 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5759 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id, 5760 adjacency_index, ecmp_size); 5761 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5762 } 5763 5764 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp, 5765 struct mlxsw_sp_fib_entry *fib_entry, 5766 enum mlxsw_reg_ralue_op op) 5767 { 5768 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif; 5769 enum mlxsw_reg_ralue_trap_action trap_action; 5770 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5771 u16 trap_id = 0; 5772 u16 rif_index = 0; 5773 5774 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) { 5775 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP; 5776 rif_index = rif->rif_index; 5777 } else { 5778 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5779 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0; 5780 } 5781 5782 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5783 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 5784 rif_index); 5785 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5786 } 5787 5788 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp, 5789 struct mlxsw_sp_fib_entry *fib_entry, 5790 enum mlxsw_reg_ralue_op op) 5791 { 5792 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5793 5794 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5795 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl); 5796 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5797 } 5798 5799 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp, 5800 struct mlxsw_sp_fib_entry *fib_entry, 5801 enum mlxsw_reg_ralue_op op) 5802 { 5803 enum mlxsw_reg_ralue_trap_action trap_action; 5804 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5805 5806 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR; 5807 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5808 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0); 5809 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5810 } 5811 5812 static int 5813 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp, 5814 struct mlxsw_sp_fib_entry *fib_entry, 5815 enum mlxsw_reg_ralue_op op) 5816 { 5817 enum mlxsw_reg_ralue_trap_action trap_action; 5818 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5819 u16 trap_id; 5820 5821 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP; 5822 trap_id = MLXSW_TRAP_ID_RTR_INGRESS1; 5823 5824 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5825 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0); 5826 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5827 } 5828 5829 static int 5830 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp, 5831 struct mlxsw_sp_fib_entry *fib_entry, 5832 enum mlxsw_reg_ralue_op op) 5833 { 5834 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry; 5835 const struct mlxsw_sp_ipip_ops *ipip_ops; 5836 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5837 int err; 5838 5839 if (WARN_ON(!ipip_entry)) 5840 return -EINVAL; 5841 5842 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]; 5843 err = ipip_ops->decap_config(mlxsw_sp, ipip_entry, 5844 fib_entry->decap.tunnel_index); 5845 if (err) 5846 return err; 5847 5848 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5849 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 5850 fib_entry->decap.tunnel_index); 5851 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5852 } 5853 5854 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp, 5855 struct mlxsw_sp_fib_entry *fib_entry, 5856 enum mlxsw_reg_ralue_op op) 5857 { 5858 char ralue_pl[MLXSW_REG_RALUE_LEN]; 5859 5860 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op); 5861 mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl, 5862 fib_entry->decap.tunnel_index); 5863 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl); 5864 } 5865 5866 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5867 struct mlxsw_sp_fib_entry *fib_entry, 5868 enum mlxsw_reg_ralue_op op) 5869 { 5870 switch (fib_entry->type) { 5871 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE: 5872 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op); 5873 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL: 5874 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op); 5875 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP: 5876 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op); 5877 case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE: 5878 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op); 5879 case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE: 5880 return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry, 5881 op); 5882 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5883 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp, 5884 fib_entry, op); 5885 case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP: 5886 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op); 5887 } 5888 return -EINVAL; 5889 } 5890 5891 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp, 5892 struct mlxsw_sp_fib_entry *fib_entry, 5893 enum mlxsw_reg_ralue_op op) 5894 { 5895 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op); 5896 5897 if (err) 5898 return err; 5899 5900 mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op); 5901 5902 return err; 5903 } 5904 5905 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp, 5906 struct mlxsw_sp_fib_entry *fib_entry) 5907 { 5908 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 5909 MLXSW_REG_RALUE_OP_WRITE_WRITE); 5910 } 5911 5912 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp, 5913 struct mlxsw_sp_fib_entry *fib_entry) 5914 { 5915 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, 5916 MLXSW_REG_RALUE_OP_WRITE_DELETE); 5917 } 5918 5919 static int 5920 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp, 5921 const struct fib_entry_notifier_info *fen_info, 5922 struct mlxsw_sp_fib_entry *fib_entry) 5923 { 5924 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 5925 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) }; 5926 struct mlxsw_sp_router *router = mlxsw_sp->router; 5927 u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id); 5928 int ifindex = nhgi->nexthops[0].ifindex; 5929 struct mlxsw_sp_ipip_entry *ipip_entry; 5930 5931 switch (fen_info->type) { 5932 case RTN_LOCAL: 5933 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 5934 MLXSW_SP_L3_PROTO_IPV4, dip); 5935 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 5936 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 5937 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, 5938 fib_entry, 5939 ipip_entry); 5940 } 5941 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 5942 MLXSW_SP_L3_PROTO_IPV4, 5943 &dip)) { 5944 u32 tunnel_index; 5945 5946 tunnel_index = router->nve_decap_config.tunnel_index; 5947 fib_entry->decap.tunnel_index = tunnel_index; 5948 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 5949 return 0; 5950 } 5951 fallthrough; 5952 case RTN_BROADCAST: 5953 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 5954 return 0; 5955 case RTN_BLACKHOLE: 5956 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 5957 return 0; 5958 case RTN_UNREACHABLE: 5959 case RTN_PROHIBIT: 5960 /* Packets hitting these routes need to be trapped, but 5961 * can do so with a lower priority than packets directed 5962 * at the host, so use action type local instead of trap. 5963 */ 5964 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 5965 return 0; 5966 case RTN_UNICAST: 5967 if (nhgi->gateway) 5968 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 5969 else 5970 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 5971 return 0; 5972 default: 5973 return -EINVAL; 5974 } 5975 } 5976 5977 static void 5978 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 5979 struct mlxsw_sp_fib_entry *fib_entry) 5980 { 5981 switch (fib_entry->type) { 5982 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP: 5983 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry); 5984 break; 5985 default: 5986 break; 5987 } 5988 } 5989 5990 static void 5991 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 5992 struct mlxsw_sp_fib4_entry *fib4_entry) 5993 { 5994 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common); 5995 } 5996 5997 static struct mlxsw_sp_fib4_entry * 5998 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp, 5999 struct mlxsw_sp_fib_node *fib_node, 6000 const struct fib_entry_notifier_info *fen_info) 6001 { 6002 struct mlxsw_sp_fib4_entry *fib4_entry; 6003 struct mlxsw_sp_fib_entry *fib_entry; 6004 int err; 6005 6006 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL); 6007 if (!fib4_entry) 6008 return ERR_PTR(-ENOMEM); 6009 fib_entry = &fib4_entry->common; 6010 6011 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi); 6012 if (err) 6013 goto err_nexthop4_group_get; 6014 6015 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6016 fib_node->fib); 6017 if (err) 6018 goto err_nexthop_group_vr_link; 6019 6020 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry); 6021 if (err) 6022 goto err_fib4_entry_type_set; 6023 6024 fib4_entry->fi = fen_info->fi; 6025 fib_info_hold(fib4_entry->fi); 6026 fib4_entry->tb_id = fen_info->tb_id; 6027 fib4_entry->type = fen_info->type; 6028 fib4_entry->dscp = fen_info->dscp; 6029 6030 fib_entry->fib_node = fib_node; 6031 6032 return fib4_entry; 6033 6034 err_fib4_entry_type_set: 6035 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6036 err_nexthop_group_vr_link: 6037 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6038 err_nexthop4_group_get: 6039 kfree(fib4_entry); 6040 return ERR_PTR(err); 6041 } 6042 6043 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6044 struct mlxsw_sp_fib4_entry *fib4_entry) 6045 { 6046 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6047 6048 fib_info_put(fib4_entry->fi); 6049 mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry); 6050 mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group, 6051 fib_node->fib); 6052 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common); 6053 kfree(fib4_entry); 6054 } 6055 6056 static struct mlxsw_sp_fib4_entry * 6057 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6058 const struct fib_entry_notifier_info *fen_info) 6059 { 6060 struct mlxsw_sp_fib4_entry *fib4_entry; 6061 struct mlxsw_sp_fib_node *fib_node; 6062 struct mlxsw_sp_fib *fib; 6063 struct mlxsw_sp_vr *vr; 6064 6065 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id); 6066 if (!vr) 6067 return NULL; 6068 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4); 6069 6070 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst, 6071 sizeof(fen_info->dst), 6072 fen_info->dst_len); 6073 if (!fib_node) 6074 return NULL; 6075 6076 fib4_entry = container_of(fib_node->fib_entry, 6077 struct mlxsw_sp_fib4_entry, common); 6078 if (fib4_entry->tb_id == fen_info->tb_id && 6079 fib4_entry->dscp == fen_info->dscp && 6080 fib4_entry->type == fen_info->type && 6081 fib4_entry->fi == fen_info->fi) 6082 return fib4_entry; 6083 6084 return NULL; 6085 } 6086 6087 static const struct rhashtable_params mlxsw_sp_fib_ht_params = { 6088 .key_offset = offsetof(struct mlxsw_sp_fib_node, key), 6089 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node), 6090 .key_len = sizeof(struct mlxsw_sp_fib_key), 6091 .automatic_shrinking = true, 6092 }; 6093 6094 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib, 6095 struct mlxsw_sp_fib_node *fib_node) 6096 { 6097 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node, 6098 mlxsw_sp_fib_ht_params); 6099 } 6100 6101 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib, 6102 struct mlxsw_sp_fib_node *fib_node) 6103 { 6104 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node, 6105 mlxsw_sp_fib_ht_params); 6106 } 6107 6108 static struct mlxsw_sp_fib_node * 6109 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr, 6110 size_t addr_len, unsigned char prefix_len) 6111 { 6112 struct mlxsw_sp_fib_key key; 6113 6114 memset(&key, 0, sizeof(key)); 6115 memcpy(key.addr, addr, addr_len); 6116 key.prefix_len = prefix_len; 6117 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params); 6118 } 6119 6120 static struct mlxsw_sp_fib_node * 6121 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr, 6122 size_t addr_len, unsigned char prefix_len) 6123 { 6124 struct mlxsw_sp_fib_node *fib_node; 6125 6126 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL); 6127 if (!fib_node) 6128 return NULL; 6129 6130 list_add(&fib_node->list, &fib->node_list); 6131 memcpy(fib_node->key.addr, addr, addr_len); 6132 fib_node->key.prefix_len = prefix_len; 6133 6134 return fib_node; 6135 } 6136 6137 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node) 6138 { 6139 list_del(&fib_node->list); 6140 kfree(fib_node); 6141 } 6142 6143 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp, 6144 struct mlxsw_sp_fib_node *fib_node) 6145 { 6146 struct mlxsw_sp_prefix_usage req_prefix_usage; 6147 struct mlxsw_sp_fib *fib = fib_node->fib; 6148 struct mlxsw_sp_lpm_tree *lpm_tree; 6149 int err; 6150 6151 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto]; 6152 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6153 goto out; 6154 6155 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6156 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len); 6157 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6158 fib->proto); 6159 if (IS_ERR(lpm_tree)) 6160 return PTR_ERR(lpm_tree); 6161 6162 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6163 if (err) 6164 goto err_lpm_tree_replace; 6165 6166 out: 6167 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++; 6168 return 0; 6169 6170 err_lpm_tree_replace: 6171 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6172 return err; 6173 } 6174 6175 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp, 6176 struct mlxsw_sp_fib_node *fib_node) 6177 { 6178 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree; 6179 struct mlxsw_sp_prefix_usage req_prefix_usage; 6180 struct mlxsw_sp_fib *fib = fib_node->fib; 6181 int err; 6182 6183 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0) 6184 return; 6185 /* Try to construct a new LPM tree from the current prefix usage 6186 * minus the unused one. If we fail, continue using the old one. 6187 */ 6188 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage); 6189 mlxsw_sp_prefix_usage_clear(&req_prefix_usage, 6190 fib_node->key.prefix_len); 6191 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage, 6192 fib->proto); 6193 if (IS_ERR(lpm_tree)) 6194 return; 6195 6196 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree); 6197 if (err) 6198 goto err_lpm_tree_replace; 6199 6200 return; 6201 6202 err_lpm_tree_replace: 6203 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree); 6204 } 6205 6206 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp, 6207 struct mlxsw_sp_fib_node *fib_node, 6208 struct mlxsw_sp_fib *fib) 6209 { 6210 int err; 6211 6212 err = mlxsw_sp_fib_node_insert(fib, fib_node); 6213 if (err) 6214 return err; 6215 fib_node->fib = fib; 6216 6217 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node); 6218 if (err) 6219 goto err_fib_lpm_tree_link; 6220 6221 return 0; 6222 6223 err_fib_lpm_tree_link: 6224 fib_node->fib = NULL; 6225 mlxsw_sp_fib_node_remove(fib, fib_node); 6226 return err; 6227 } 6228 6229 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp, 6230 struct mlxsw_sp_fib_node *fib_node) 6231 { 6232 struct mlxsw_sp_fib *fib = fib_node->fib; 6233 6234 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node); 6235 fib_node->fib = NULL; 6236 mlxsw_sp_fib_node_remove(fib, fib_node); 6237 } 6238 6239 static struct mlxsw_sp_fib_node * 6240 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr, 6241 size_t addr_len, unsigned char prefix_len, 6242 enum mlxsw_sp_l3proto proto) 6243 { 6244 struct mlxsw_sp_fib_node *fib_node; 6245 struct mlxsw_sp_fib *fib; 6246 struct mlxsw_sp_vr *vr; 6247 int err; 6248 6249 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL); 6250 if (IS_ERR(vr)) 6251 return ERR_CAST(vr); 6252 fib = mlxsw_sp_vr_fib(vr, proto); 6253 6254 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len); 6255 if (fib_node) 6256 return fib_node; 6257 6258 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len); 6259 if (!fib_node) { 6260 err = -ENOMEM; 6261 goto err_fib_node_create; 6262 } 6263 6264 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib); 6265 if (err) 6266 goto err_fib_node_init; 6267 6268 return fib_node; 6269 6270 err_fib_node_init: 6271 mlxsw_sp_fib_node_destroy(fib_node); 6272 err_fib_node_create: 6273 mlxsw_sp_vr_put(mlxsw_sp, vr); 6274 return ERR_PTR(err); 6275 } 6276 6277 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp, 6278 struct mlxsw_sp_fib_node *fib_node) 6279 { 6280 struct mlxsw_sp_vr *vr = fib_node->fib->vr; 6281 6282 if (fib_node->fib_entry) 6283 return; 6284 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node); 6285 mlxsw_sp_fib_node_destroy(fib_node); 6286 mlxsw_sp_vr_put(mlxsw_sp, vr); 6287 } 6288 6289 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp, 6290 struct mlxsw_sp_fib_entry *fib_entry) 6291 { 6292 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6293 int err; 6294 6295 fib_node->fib_entry = fib_entry; 6296 6297 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry); 6298 if (err) 6299 goto err_fib_entry_update; 6300 6301 return 0; 6302 6303 err_fib_entry_update: 6304 fib_node->fib_entry = NULL; 6305 return err; 6306 } 6307 6308 static void 6309 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp, 6310 struct mlxsw_sp_fib_entry *fib_entry) 6311 { 6312 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node; 6313 6314 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); 6315 fib_node->fib_entry = NULL; 6316 } 6317 6318 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry) 6319 { 6320 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node; 6321 struct mlxsw_sp_fib4_entry *fib4_replaced; 6322 6323 if (!fib_node->fib_entry) 6324 return true; 6325 6326 fib4_replaced = container_of(fib_node->fib_entry, 6327 struct mlxsw_sp_fib4_entry, common); 6328 if (fib4_entry->tb_id == RT_TABLE_MAIN && 6329 fib4_replaced->tb_id == RT_TABLE_LOCAL) 6330 return false; 6331 6332 return true; 6333 } 6334 6335 static int 6336 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp, 6337 const struct fib_entry_notifier_info *fen_info) 6338 { 6339 struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced; 6340 struct mlxsw_sp_fib_entry *replaced; 6341 struct mlxsw_sp_fib_node *fib_node; 6342 int err; 6343 6344 if (fen_info->fi->nh && 6345 !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id)) 6346 return 0; 6347 6348 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id, 6349 &fen_info->dst, sizeof(fen_info->dst), 6350 fen_info->dst_len, 6351 MLXSW_SP_L3_PROTO_IPV4); 6352 if (IS_ERR(fib_node)) { 6353 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n"); 6354 return PTR_ERR(fib_node); 6355 } 6356 6357 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info); 6358 if (IS_ERR(fib4_entry)) { 6359 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n"); 6360 err = PTR_ERR(fib4_entry); 6361 goto err_fib4_entry_create; 6362 } 6363 6364 if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) { 6365 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6366 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6367 return 0; 6368 } 6369 6370 replaced = fib_node->fib_entry; 6371 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common); 6372 if (err) { 6373 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n"); 6374 goto err_fib_node_entry_link; 6375 } 6376 6377 /* Nothing to replace */ 6378 if (!replaced) 6379 return 0; 6380 6381 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 6382 fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry, 6383 common); 6384 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced); 6385 6386 return 0; 6387 6388 err_fib_node_entry_link: 6389 fib_node->fib_entry = replaced; 6390 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6391 err_fib4_entry_create: 6392 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6393 return err; 6394 } 6395 6396 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, 6397 struct fib_entry_notifier_info *fen_info) 6398 { 6399 struct mlxsw_sp_fib4_entry *fib4_entry; 6400 struct mlxsw_sp_fib_node *fib_node; 6401 6402 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info); 6403 if (!fib4_entry) 6404 return; 6405 fib_node = fib4_entry->common.fib_node; 6406 6407 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common); 6408 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 6409 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 6410 } 6411 6412 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt) 6413 { 6414 /* Multicast routes aren't supported, so ignore them. Neighbour 6415 * Discovery packets are specifically trapped. 6416 */ 6417 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST) 6418 return true; 6419 6420 /* Cloned routes are irrelevant in the forwarding path. */ 6421 if (rt->fib6_flags & RTF_CACHE) 6422 return true; 6423 6424 return false; 6425 } 6426 6427 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt) 6428 { 6429 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6430 6431 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL); 6432 if (!mlxsw_sp_rt6) 6433 return ERR_PTR(-ENOMEM); 6434 6435 /* In case of route replace, replaced route is deleted with 6436 * no notification. Take reference to prevent accessing freed 6437 * memory. 6438 */ 6439 mlxsw_sp_rt6->rt = rt; 6440 fib6_info_hold(rt); 6441 6442 return mlxsw_sp_rt6; 6443 } 6444 6445 #if IS_ENABLED(CONFIG_IPV6) 6446 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6447 { 6448 fib6_info_release(rt); 6449 } 6450 #else 6451 static void mlxsw_sp_rt6_release(struct fib6_info *rt) 6452 { 6453 } 6454 #endif 6455 6456 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6) 6457 { 6458 struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh; 6459 6460 if (!mlxsw_sp_rt6->rt->nh) 6461 fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD; 6462 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt); 6463 kfree(mlxsw_sp_rt6); 6464 } 6465 6466 static struct fib6_info * 6467 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry) 6468 { 6469 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6, 6470 list)->rt; 6471 } 6472 6473 static struct mlxsw_sp_rt6 * 6474 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry, 6475 const struct fib6_info *rt) 6476 { 6477 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6478 6479 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) { 6480 if (mlxsw_sp_rt6->rt == rt) 6481 return mlxsw_sp_rt6; 6482 } 6483 6484 return NULL; 6485 } 6486 6487 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp, 6488 const struct fib6_info *rt, 6489 enum mlxsw_sp_ipip_type *ret) 6490 { 6491 return rt->fib6_nh->fib_nh_dev && 6492 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret); 6493 } 6494 6495 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp, 6496 struct mlxsw_sp_nexthop_group *nh_grp, 6497 struct mlxsw_sp_nexthop *nh, 6498 const struct fib6_info *rt) 6499 { 6500 struct net_device *dev = rt->fib6_nh->fib_nh_dev; 6501 6502 nh->nhgi = nh_grp->nhgi; 6503 nh->nh_weight = rt->fib6_nh->fib_nh_weight; 6504 memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr)); 6505 #if IS_ENABLED(CONFIG_IPV6) 6506 nh->neigh_tbl = &nd_tbl; 6507 #endif 6508 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh); 6509 6510 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list); 6511 6512 if (!dev) 6513 return 0; 6514 nh->ifindex = dev->ifindex; 6515 6516 return mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev); 6517 } 6518 6519 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp, 6520 struct mlxsw_sp_nexthop *nh) 6521 { 6522 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh); 6523 list_del(&nh->router_list_node); 6524 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh); 6525 } 6526 6527 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp, 6528 const struct fib6_info *rt) 6529 { 6530 return rt->fib6_nh->fib_nh_gw_family || 6531 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL); 6532 } 6533 6534 static int 6535 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp, 6536 struct mlxsw_sp_nexthop_group *nh_grp, 6537 struct mlxsw_sp_fib6_entry *fib6_entry) 6538 { 6539 struct mlxsw_sp_nexthop_group_info *nhgi; 6540 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6541 struct mlxsw_sp_nexthop *nh; 6542 int err, i; 6543 6544 nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6), 6545 GFP_KERNEL); 6546 if (!nhgi) 6547 return -ENOMEM; 6548 nh_grp->nhgi = nhgi; 6549 nhgi->nh_grp = nh_grp; 6550 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list, 6551 struct mlxsw_sp_rt6, list); 6552 nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt); 6553 nhgi->count = fib6_entry->nrt6; 6554 for (i = 0; i < nhgi->count; i++) { 6555 struct fib6_info *rt = mlxsw_sp_rt6->rt; 6556 6557 nh = &nhgi->nexthops[i]; 6558 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt); 6559 if (err) 6560 goto err_nexthop6_init; 6561 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list); 6562 } 6563 nh_grp->nhgi = nhgi; 6564 err = mlxsw_sp_nexthop_group_inc(mlxsw_sp); 6565 if (err) 6566 goto err_group_inc; 6567 err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6568 if (err) 6569 goto err_group_refresh; 6570 6571 return 0; 6572 6573 err_group_refresh: 6574 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6575 err_group_inc: 6576 i = nhgi->count; 6577 err_nexthop6_init: 6578 for (i--; i >= 0; i--) { 6579 nh = &nhgi->nexthops[i]; 6580 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6581 } 6582 kfree(nhgi); 6583 return err; 6584 } 6585 6586 static void 6587 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp, 6588 struct mlxsw_sp_nexthop_group *nh_grp) 6589 { 6590 struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi; 6591 int i; 6592 6593 mlxsw_sp_nexthop_group_dec(mlxsw_sp); 6594 for (i = nhgi->count - 1; i >= 0; i--) { 6595 struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i]; 6596 6597 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh); 6598 } 6599 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp); 6600 WARN_ON_ONCE(nhgi->adj_index_valid); 6601 kfree(nhgi); 6602 } 6603 6604 static struct mlxsw_sp_nexthop_group * 6605 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp, 6606 struct mlxsw_sp_fib6_entry *fib6_entry) 6607 { 6608 struct mlxsw_sp_nexthop_group *nh_grp; 6609 int err; 6610 6611 nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL); 6612 if (!nh_grp) 6613 return ERR_PTR(-ENOMEM); 6614 INIT_LIST_HEAD(&nh_grp->vr_list); 6615 err = rhashtable_init(&nh_grp->vr_ht, 6616 &mlxsw_sp_nexthop_group_vr_ht_params); 6617 if (err) 6618 goto err_nexthop_group_vr_ht_init; 6619 INIT_LIST_HEAD(&nh_grp->fib_list); 6620 nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6; 6621 6622 err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry); 6623 if (err) 6624 goto err_nexthop_group_info_init; 6625 6626 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp); 6627 if (err) 6628 goto err_nexthop_group_insert; 6629 6630 nh_grp->can_destroy = true; 6631 6632 return nh_grp; 6633 6634 err_nexthop_group_insert: 6635 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6636 err_nexthop_group_info_init: 6637 rhashtable_destroy(&nh_grp->vr_ht); 6638 err_nexthop_group_vr_ht_init: 6639 kfree(nh_grp); 6640 return ERR_PTR(err); 6641 } 6642 6643 static void 6644 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp, 6645 struct mlxsw_sp_nexthop_group *nh_grp) 6646 { 6647 if (!nh_grp->can_destroy) 6648 return; 6649 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp); 6650 mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp); 6651 WARN_ON_ONCE(!list_empty(&nh_grp->vr_list)); 6652 rhashtable_destroy(&nh_grp->vr_ht); 6653 kfree(nh_grp); 6654 } 6655 6656 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp, 6657 struct mlxsw_sp_fib6_entry *fib6_entry) 6658 { 6659 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6660 struct mlxsw_sp_nexthop_group *nh_grp; 6661 6662 if (rt->nh) { 6663 nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, 6664 rt->nh->id); 6665 if (WARN_ON_ONCE(!nh_grp)) 6666 return -EINVAL; 6667 goto out; 6668 } 6669 6670 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry); 6671 if (!nh_grp) { 6672 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry); 6673 if (IS_ERR(nh_grp)) 6674 return PTR_ERR(nh_grp); 6675 } 6676 6677 /* The route and the nexthop are described by the same struct, so we 6678 * need to the update the nexthop offload indication for the new route. 6679 */ 6680 __mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry); 6681 6682 out: 6683 list_add_tail(&fib6_entry->common.nexthop_group_node, 6684 &nh_grp->fib_list); 6685 fib6_entry->common.nh_group = nh_grp; 6686 6687 return 0; 6688 } 6689 6690 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp, 6691 struct mlxsw_sp_fib_entry *fib_entry) 6692 { 6693 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group; 6694 6695 list_del(&fib_entry->nexthop_group_node); 6696 if (!list_empty(&nh_grp->fib_list)) 6697 return; 6698 6699 if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) { 6700 mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp); 6701 return; 6702 } 6703 6704 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp); 6705 } 6706 6707 static int 6708 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp, 6709 struct mlxsw_sp_fib6_entry *fib6_entry) 6710 { 6711 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group; 6712 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6713 int err; 6714 6715 mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib); 6716 fib6_entry->common.nh_group = NULL; 6717 list_del(&fib6_entry->common.nexthop_group_node); 6718 6719 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6720 if (err) 6721 goto err_nexthop6_group_get; 6722 6723 err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group, 6724 fib_node->fib); 6725 if (err) 6726 goto err_nexthop_group_vr_link; 6727 6728 /* In case this entry is offloaded, then the adjacency index 6729 * currently associated with it in the device's table is that 6730 * of the old group. Start using the new one instead. 6731 */ 6732 err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common); 6733 if (err) 6734 goto err_fib_entry_update; 6735 6736 if (list_empty(&old_nh_grp->fib_list)) 6737 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp); 6738 6739 return 0; 6740 6741 err_fib_entry_update: 6742 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6743 fib_node->fib); 6744 err_nexthop_group_vr_link: 6745 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6746 err_nexthop6_group_get: 6747 list_add_tail(&fib6_entry->common.nexthop_group_node, 6748 &old_nh_grp->fib_list); 6749 fib6_entry->common.nh_group = old_nh_grp; 6750 mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib); 6751 return err; 6752 } 6753 6754 static int 6755 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp, 6756 struct mlxsw_sp_fib6_entry *fib6_entry, 6757 struct fib6_info **rt_arr, unsigned int nrt6) 6758 { 6759 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6760 int err, i; 6761 6762 for (i = 0; i < nrt6; i++) { 6763 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6764 if (IS_ERR(mlxsw_sp_rt6)) { 6765 err = PTR_ERR(mlxsw_sp_rt6); 6766 goto err_rt6_unwind; 6767 } 6768 6769 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6770 fib6_entry->nrt6++; 6771 } 6772 6773 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 6774 if (err) 6775 goto err_rt6_unwind; 6776 6777 return 0; 6778 6779 err_rt6_unwind: 6780 for (; i > 0; i--) { 6781 fib6_entry->nrt6--; 6782 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6783 struct mlxsw_sp_rt6, list); 6784 list_del(&mlxsw_sp_rt6->list); 6785 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6786 } 6787 return err; 6788 } 6789 6790 static void 6791 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp, 6792 struct mlxsw_sp_fib6_entry *fib6_entry, 6793 struct fib6_info **rt_arr, unsigned int nrt6) 6794 { 6795 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6796 int i; 6797 6798 for (i = 0; i < nrt6; i++) { 6799 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, 6800 rt_arr[i]); 6801 if (WARN_ON_ONCE(!mlxsw_sp_rt6)) 6802 continue; 6803 6804 fib6_entry->nrt6--; 6805 list_del(&mlxsw_sp_rt6->list); 6806 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6807 } 6808 6809 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry); 6810 } 6811 6812 static int 6813 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp, 6814 struct mlxsw_sp_fib_entry *fib_entry, 6815 const struct fib6_info *rt) 6816 { 6817 struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi; 6818 union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr }; 6819 u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id); 6820 struct mlxsw_sp_router *router = mlxsw_sp->router; 6821 int ifindex = nhgi->nexthops[0].ifindex; 6822 struct mlxsw_sp_ipip_entry *ipip_entry; 6823 6824 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6825 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex, 6826 MLXSW_SP_L3_PROTO_IPV6, 6827 dip); 6828 6829 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) { 6830 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP; 6831 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry, 6832 ipip_entry); 6833 } 6834 if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id, 6835 MLXSW_SP_L3_PROTO_IPV6, &dip)) { 6836 u32 tunnel_index; 6837 6838 tunnel_index = router->nve_decap_config.tunnel_index; 6839 fib_entry->decap.tunnel_index = tunnel_index; 6840 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP; 6841 } 6842 6843 return 0; 6844 } 6845 6846 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp, 6847 struct mlxsw_sp_fib_entry *fib_entry, 6848 const struct fib6_info *rt) 6849 { 6850 if (rt->fib6_flags & RTF_LOCAL) 6851 return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry, 6852 rt); 6853 if (rt->fib6_flags & RTF_ANYCAST) 6854 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP; 6855 else if (rt->fib6_type == RTN_BLACKHOLE) 6856 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE; 6857 else if (rt->fib6_flags & RTF_REJECT) 6858 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE; 6859 else if (fib_entry->nh_group->nhgi->gateway) 6860 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE; 6861 else 6862 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL; 6863 6864 return 0; 6865 } 6866 6867 static void 6868 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry) 6869 { 6870 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp; 6871 6872 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list, 6873 list) { 6874 fib6_entry->nrt6--; 6875 list_del(&mlxsw_sp_rt6->list); 6876 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6877 } 6878 } 6879 6880 static struct mlxsw_sp_fib6_entry * 6881 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp, 6882 struct mlxsw_sp_fib_node *fib_node, 6883 struct fib6_info **rt_arr, unsigned int nrt6) 6884 { 6885 struct mlxsw_sp_fib6_entry *fib6_entry; 6886 struct mlxsw_sp_fib_entry *fib_entry; 6887 struct mlxsw_sp_rt6 *mlxsw_sp_rt6; 6888 int err, i; 6889 6890 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL); 6891 if (!fib6_entry) 6892 return ERR_PTR(-ENOMEM); 6893 fib_entry = &fib6_entry->common; 6894 6895 INIT_LIST_HEAD(&fib6_entry->rt6_list); 6896 6897 for (i = 0; i < nrt6; i++) { 6898 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]); 6899 if (IS_ERR(mlxsw_sp_rt6)) { 6900 err = PTR_ERR(mlxsw_sp_rt6); 6901 goto err_rt6_unwind; 6902 } 6903 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list); 6904 fib6_entry->nrt6++; 6905 } 6906 6907 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry); 6908 if (err) 6909 goto err_rt6_unwind; 6910 6911 err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group, 6912 fib_node->fib); 6913 if (err) 6914 goto err_nexthop_group_vr_link; 6915 6916 err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]); 6917 if (err) 6918 goto err_fib6_entry_type_set; 6919 6920 fib_entry->fib_node = fib_node; 6921 6922 return fib6_entry; 6923 6924 err_fib6_entry_type_set: 6925 mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib); 6926 err_nexthop_group_vr_link: 6927 mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry); 6928 err_rt6_unwind: 6929 for (; i > 0; i--) { 6930 fib6_entry->nrt6--; 6931 mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list, 6932 struct mlxsw_sp_rt6, list); 6933 list_del(&mlxsw_sp_rt6->list); 6934 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6); 6935 } 6936 kfree(fib6_entry); 6937 return ERR_PTR(err); 6938 } 6939 6940 static void 6941 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp, 6942 struct mlxsw_sp_fib6_entry *fib6_entry) 6943 { 6944 mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common); 6945 } 6946 6947 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp, 6948 struct mlxsw_sp_fib6_entry *fib6_entry) 6949 { 6950 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6951 6952 mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry); 6953 mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group, 6954 fib_node->fib); 6955 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common); 6956 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry); 6957 WARN_ON(fib6_entry->nrt6); 6958 kfree(fib6_entry); 6959 } 6960 6961 static struct mlxsw_sp_fib6_entry * 6962 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp, 6963 const struct fib6_info *rt) 6964 { 6965 struct mlxsw_sp_fib6_entry *fib6_entry; 6966 struct mlxsw_sp_fib_node *fib_node; 6967 struct mlxsw_sp_fib *fib; 6968 struct fib6_info *cmp_rt; 6969 struct mlxsw_sp_vr *vr; 6970 6971 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id); 6972 if (!vr) 6973 return NULL; 6974 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6); 6975 6976 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr, 6977 sizeof(rt->fib6_dst.addr), 6978 rt->fib6_dst.plen); 6979 if (!fib_node) 6980 return NULL; 6981 6982 fib6_entry = container_of(fib_node->fib_entry, 6983 struct mlxsw_sp_fib6_entry, common); 6984 cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 6985 if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id && 6986 rt->fib6_metric == cmp_rt->fib6_metric && 6987 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt)) 6988 return fib6_entry; 6989 6990 return NULL; 6991 } 6992 6993 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry) 6994 { 6995 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node; 6996 struct mlxsw_sp_fib6_entry *fib6_replaced; 6997 struct fib6_info *rt, *rt_replaced; 6998 6999 if (!fib_node->fib_entry) 7000 return true; 7001 7002 fib6_replaced = container_of(fib_node->fib_entry, 7003 struct mlxsw_sp_fib6_entry, 7004 common); 7005 rt = mlxsw_sp_fib6_entry_rt(fib6_entry); 7006 rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced); 7007 if (rt->fib6_table->tb6_id == RT_TABLE_MAIN && 7008 rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL) 7009 return false; 7010 7011 return true; 7012 } 7013 7014 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp, 7015 struct fib6_info **rt_arr, 7016 unsigned int nrt6) 7017 { 7018 struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced; 7019 struct mlxsw_sp_fib_entry *replaced; 7020 struct mlxsw_sp_fib_node *fib_node; 7021 struct fib6_info *rt = rt_arr[0]; 7022 int err; 7023 7024 if (rt->fib6_src.plen) 7025 return -EINVAL; 7026 7027 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7028 return 0; 7029 7030 if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id)) 7031 return 0; 7032 7033 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7034 &rt->fib6_dst.addr, 7035 sizeof(rt->fib6_dst.addr), 7036 rt->fib6_dst.plen, 7037 MLXSW_SP_L3_PROTO_IPV6); 7038 if (IS_ERR(fib_node)) 7039 return PTR_ERR(fib_node); 7040 7041 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr, 7042 nrt6); 7043 if (IS_ERR(fib6_entry)) { 7044 err = PTR_ERR(fib6_entry); 7045 goto err_fib6_entry_create; 7046 } 7047 7048 if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) { 7049 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7050 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7051 return 0; 7052 } 7053 7054 replaced = fib_node->fib_entry; 7055 err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common); 7056 if (err) 7057 goto err_fib_node_entry_link; 7058 7059 /* Nothing to replace */ 7060 if (!replaced) 7061 return 0; 7062 7063 mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced); 7064 fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry, 7065 common); 7066 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced); 7067 7068 return 0; 7069 7070 err_fib_node_entry_link: 7071 fib_node->fib_entry = replaced; 7072 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7073 err_fib6_entry_create: 7074 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7075 return err; 7076 } 7077 7078 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp, 7079 struct fib6_info **rt_arr, 7080 unsigned int nrt6) 7081 { 7082 struct mlxsw_sp_fib6_entry *fib6_entry; 7083 struct mlxsw_sp_fib_node *fib_node; 7084 struct fib6_info *rt = rt_arr[0]; 7085 int err; 7086 7087 if (rt->fib6_src.plen) 7088 return -EINVAL; 7089 7090 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7091 return 0; 7092 7093 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id, 7094 &rt->fib6_dst.addr, 7095 sizeof(rt->fib6_dst.addr), 7096 rt->fib6_dst.plen, 7097 MLXSW_SP_L3_PROTO_IPV6); 7098 if (IS_ERR(fib_node)) 7099 return PTR_ERR(fib_node); 7100 7101 if (WARN_ON_ONCE(!fib_node->fib_entry)) { 7102 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7103 return -EINVAL; 7104 } 7105 7106 fib6_entry = container_of(fib_node->fib_entry, 7107 struct mlxsw_sp_fib6_entry, common); 7108 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr, 7109 nrt6); 7110 if (err) 7111 goto err_fib6_entry_nexthop_add; 7112 7113 return 0; 7114 7115 err_fib6_entry_nexthop_add: 7116 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7117 return err; 7118 } 7119 7120 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp, 7121 struct fib6_info **rt_arr, 7122 unsigned int nrt6) 7123 { 7124 struct mlxsw_sp_fib6_entry *fib6_entry; 7125 struct mlxsw_sp_fib_node *fib_node; 7126 struct fib6_info *rt = rt_arr[0]; 7127 7128 if (mlxsw_sp_fib6_rt_should_ignore(rt)) 7129 return; 7130 7131 /* Multipath routes are first added to the FIB trie and only then 7132 * notified. If we vetoed the addition, we will get a delete 7133 * notification for a route we do not have. Therefore, do not warn if 7134 * route was not found. 7135 */ 7136 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt); 7137 if (!fib6_entry) 7138 return; 7139 7140 /* If not all the nexthops are deleted, then only reduce the nexthop 7141 * group. 7142 */ 7143 if (nrt6 != fib6_entry->nrt6) { 7144 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr, 7145 nrt6); 7146 return; 7147 } 7148 7149 fib_node = fib6_entry->common.fib_node; 7150 7151 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common); 7152 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7153 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7154 } 7155 7156 static struct mlxsw_sp_mr_table * 7157 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family) 7158 { 7159 if (family == RTNL_FAMILY_IPMR) 7160 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]; 7161 else 7162 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]; 7163 } 7164 7165 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp, 7166 struct mfc_entry_notifier_info *men_info, 7167 bool replace) 7168 { 7169 struct mlxsw_sp_mr_table *mrt; 7170 struct mlxsw_sp_vr *vr; 7171 7172 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL); 7173 if (IS_ERR(vr)) 7174 return PTR_ERR(vr); 7175 7176 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7177 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace); 7178 } 7179 7180 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp, 7181 struct mfc_entry_notifier_info *men_info) 7182 { 7183 struct mlxsw_sp_mr_table *mrt; 7184 struct mlxsw_sp_vr *vr; 7185 7186 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id); 7187 if (WARN_ON(!vr)) 7188 return; 7189 7190 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family); 7191 mlxsw_sp_mr_route_del(mrt, men_info->mfc); 7192 mlxsw_sp_vr_put(mlxsw_sp, vr); 7193 } 7194 7195 static int 7196 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp, 7197 struct vif_entry_notifier_info *ven_info) 7198 { 7199 struct mlxsw_sp_mr_table *mrt; 7200 struct mlxsw_sp_rif *rif; 7201 struct mlxsw_sp_vr *vr; 7202 7203 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL); 7204 if (IS_ERR(vr)) 7205 return PTR_ERR(vr); 7206 7207 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7208 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev); 7209 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev, 7210 ven_info->vif_index, 7211 ven_info->vif_flags, rif); 7212 } 7213 7214 static void 7215 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp, 7216 struct vif_entry_notifier_info *ven_info) 7217 { 7218 struct mlxsw_sp_mr_table *mrt; 7219 struct mlxsw_sp_vr *vr; 7220 7221 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id); 7222 if (WARN_ON(!vr)) 7223 return; 7224 7225 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family); 7226 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index); 7227 mlxsw_sp_vr_put(mlxsw_sp, vr); 7228 } 7229 7230 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp, 7231 struct mlxsw_sp_fib_node *fib_node) 7232 { 7233 struct mlxsw_sp_fib4_entry *fib4_entry; 7234 7235 fib4_entry = container_of(fib_node->fib_entry, 7236 struct mlxsw_sp_fib4_entry, common); 7237 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7238 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry); 7239 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7240 } 7241 7242 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp, 7243 struct mlxsw_sp_fib_node *fib_node) 7244 { 7245 struct mlxsw_sp_fib6_entry *fib6_entry; 7246 7247 fib6_entry = container_of(fib_node->fib_entry, 7248 struct mlxsw_sp_fib6_entry, common); 7249 mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry); 7250 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry); 7251 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node); 7252 } 7253 7254 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp, 7255 struct mlxsw_sp_fib_node *fib_node) 7256 { 7257 switch (fib_node->fib->proto) { 7258 case MLXSW_SP_L3_PROTO_IPV4: 7259 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node); 7260 break; 7261 case MLXSW_SP_L3_PROTO_IPV6: 7262 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node); 7263 break; 7264 } 7265 } 7266 7267 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp, 7268 struct mlxsw_sp_vr *vr, 7269 enum mlxsw_sp_l3proto proto) 7270 { 7271 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto); 7272 struct mlxsw_sp_fib_node *fib_node, *tmp; 7273 7274 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) { 7275 bool do_break = &tmp->list == &fib->node_list; 7276 7277 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node); 7278 if (do_break) 7279 break; 7280 } 7281 } 7282 7283 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp) 7284 { 7285 int i, j; 7286 7287 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) { 7288 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i]; 7289 7290 if (!mlxsw_sp_vr_is_used(vr)) 7291 continue; 7292 7293 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++) 7294 mlxsw_sp_mr_table_flush(vr->mr_table[j]); 7295 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4); 7296 7297 /* If virtual router was only used for IPv4, then it's no 7298 * longer used. 7299 */ 7300 if (!mlxsw_sp_vr_is_used(vr)) 7301 continue; 7302 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6); 7303 } 7304 } 7305 7306 struct mlxsw_sp_fib6_event_work { 7307 struct fib6_info **rt_arr; 7308 unsigned int nrt6; 7309 }; 7310 7311 struct mlxsw_sp_fib_event_work { 7312 struct work_struct work; 7313 union { 7314 struct mlxsw_sp_fib6_event_work fib6_work; 7315 struct fib_entry_notifier_info fen_info; 7316 struct fib_rule_notifier_info fr_info; 7317 struct fib_nh_notifier_info fnh_info; 7318 struct mfc_entry_notifier_info men_info; 7319 struct vif_entry_notifier_info ven_info; 7320 }; 7321 struct mlxsw_sp *mlxsw_sp; 7322 unsigned long event; 7323 }; 7324 7325 static int 7326 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work, 7327 struct fib6_entry_notifier_info *fen6_info) 7328 { 7329 struct fib6_info *rt = fen6_info->rt; 7330 struct fib6_info **rt_arr; 7331 struct fib6_info *iter; 7332 unsigned int nrt6; 7333 int i = 0; 7334 7335 nrt6 = fen6_info->nsiblings + 1; 7336 7337 rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC); 7338 if (!rt_arr) 7339 return -ENOMEM; 7340 7341 fib6_work->rt_arr = rt_arr; 7342 fib6_work->nrt6 = nrt6; 7343 7344 rt_arr[0] = rt; 7345 fib6_info_hold(rt); 7346 7347 if (!fen6_info->nsiblings) 7348 return 0; 7349 7350 list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) { 7351 if (i == fen6_info->nsiblings) 7352 break; 7353 7354 rt_arr[i + 1] = iter; 7355 fib6_info_hold(iter); 7356 i++; 7357 } 7358 WARN_ON_ONCE(i != fen6_info->nsiblings); 7359 7360 return 0; 7361 } 7362 7363 static void 7364 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work) 7365 { 7366 int i; 7367 7368 for (i = 0; i < fib6_work->nrt6; i++) 7369 mlxsw_sp_rt6_release(fib6_work->rt_arr[i]); 7370 kfree(fib6_work->rt_arr); 7371 } 7372 7373 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work) 7374 { 7375 struct mlxsw_sp_fib_event_work *fib_work = 7376 container_of(work, struct mlxsw_sp_fib_event_work, work); 7377 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7378 int err; 7379 7380 mutex_lock(&mlxsw_sp->router->lock); 7381 mlxsw_sp_span_respin(mlxsw_sp); 7382 7383 switch (fib_work->event) { 7384 case FIB_EVENT_ENTRY_REPLACE: 7385 err = mlxsw_sp_router_fib4_replace(mlxsw_sp, 7386 &fib_work->fen_info); 7387 if (err) { 7388 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7389 mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp, 7390 &fib_work->fen_info); 7391 } 7392 fib_info_put(fib_work->fen_info.fi); 7393 break; 7394 case FIB_EVENT_ENTRY_DEL: 7395 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info); 7396 fib_info_put(fib_work->fen_info.fi); 7397 break; 7398 case FIB_EVENT_NH_ADD: 7399 case FIB_EVENT_NH_DEL: 7400 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event, 7401 fib_work->fnh_info.fib_nh); 7402 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent); 7403 break; 7404 } 7405 mutex_unlock(&mlxsw_sp->router->lock); 7406 kfree(fib_work); 7407 } 7408 7409 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work) 7410 { 7411 struct mlxsw_sp_fib_event_work *fib_work = 7412 container_of(work, struct mlxsw_sp_fib_event_work, work); 7413 struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work; 7414 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7415 int err; 7416 7417 mutex_lock(&mlxsw_sp->router->lock); 7418 mlxsw_sp_span_respin(mlxsw_sp); 7419 7420 switch (fib_work->event) { 7421 case FIB_EVENT_ENTRY_REPLACE: 7422 err = mlxsw_sp_router_fib6_replace(mlxsw_sp, 7423 fib6_work->rt_arr, 7424 fib6_work->nrt6); 7425 if (err) { 7426 dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n"); 7427 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7428 fib6_work->rt_arr, 7429 fib6_work->nrt6); 7430 } 7431 mlxsw_sp_router_fib6_work_fini(fib6_work); 7432 break; 7433 case FIB_EVENT_ENTRY_APPEND: 7434 err = mlxsw_sp_router_fib6_append(mlxsw_sp, 7435 fib6_work->rt_arr, 7436 fib6_work->nrt6); 7437 if (err) { 7438 dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n"); 7439 mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp, 7440 fib6_work->rt_arr, 7441 fib6_work->nrt6); 7442 } 7443 mlxsw_sp_router_fib6_work_fini(fib6_work); 7444 break; 7445 case FIB_EVENT_ENTRY_DEL: 7446 mlxsw_sp_router_fib6_del(mlxsw_sp, 7447 fib6_work->rt_arr, 7448 fib6_work->nrt6); 7449 mlxsw_sp_router_fib6_work_fini(fib6_work); 7450 break; 7451 } 7452 mutex_unlock(&mlxsw_sp->router->lock); 7453 kfree(fib_work); 7454 } 7455 7456 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work) 7457 { 7458 struct mlxsw_sp_fib_event_work *fib_work = 7459 container_of(work, struct mlxsw_sp_fib_event_work, work); 7460 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp; 7461 bool replace; 7462 int err; 7463 7464 rtnl_lock(); 7465 mutex_lock(&mlxsw_sp->router->lock); 7466 switch (fib_work->event) { 7467 case FIB_EVENT_ENTRY_REPLACE: 7468 case FIB_EVENT_ENTRY_ADD: 7469 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE; 7470 7471 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info, 7472 replace); 7473 if (err) 7474 dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n"); 7475 mr_cache_put(fib_work->men_info.mfc); 7476 break; 7477 case FIB_EVENT_ENTRY_DEL: 7478 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info); 7479 mr_cache_put(fib_work->men_info.mfc); 7480 break; 7481 case FIB_EVENT_VIF_ADD: 7482 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp, 7483 &fib_work->ven_info); 7484 if (err) 7485 dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n"); 7486 dev_put(fib_work->ven_info.dev); 7487 break; 7488 case FIB_EVENT_VIF_DEL: 7489 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp, 7490 &fib_work->ven_info); 7491 dev_put(fib_work->ven_info.dev); 7492 break; 7493 } 7494 mutex_unlock(&mlxsw_sp->router->lock); 7495 rtnl_unlock(); 7496 kfree(fib_work); 7497 } 7498 7499 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work, 7500 struct fib_notifier_info *info) 7501 { 7502 struct fib_entry_notifier_info *fen_info; 7503 struct fib_nh_notifier_info *fnh_info; 7504 7505 switch (fib_work->event) { 7506 case FIB_EVENT_ENTRY_REPLACE: 7507 case FIB_EVENT_ENTRY_DEL: 7508 fen_info = container_of(info, struct fib_entry_notifier_info, 7509 info); 7510 fib_work->fen_info = *fen_info; 7511 /* Take reference on fib_info to prevent it from being 7512 * freed while work is queued. Release it afterwards. 7513 */ 7514 fib_info_hold(fib_work->fen_info.fi); 7515 break; 7516 case FIB_EVENT_NH_ADD: 7517 case FIB_EVENT_NH_DEL: 7518 fnh_info = container_of(info, struct fib_nh_notifier_info, 7519 info); 7520 fib_work->fnh_info = *fnh_info; 7521 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent); 7522 break; 7523 } 7524 } 7525 7526 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work, 7527 struct fib_notifier_info *info) 7528 { 7529 struct fib6_entry_notifier_info *fen6_info; 7530 int err; 7531 7532 switch (fib_work->event) { 7533 case FIB_EVENT_ENTRY_REPLACE: 7534 case FIB_EVENT_ENTRY_APPEND: 7535 case FIB_EVENT_ENTRY_DEL: 7536 fen6_info = container_of(info, struct fib6_entry_notifier_info, 7537 info); 7538 err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work, 7539 fen6_info); 7540 if (err) 7541 return err; 7542 break; 7543 } 7544 7545 return 0; 7546 } 7547 7548 static void 7549 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work, 7550 struct fib_notifier_info *info) 7551 { 7552 switch (fib_work->event) { 7553 case FIB_EVENT_ENTRY_REPLACE: 7554 case FIB_EVENT_ENTRY_ADD: 7555 case FIB_EVENT_ENTRY_DEL: 7556 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info)); 7557 mr_cache_hold(fib_work->men_info.mfc); 7558 break; 7559 case FIB_EVENT_VIF_ADD: 7560 case FIB_EVENT_VIF_DEL: 7561 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info)); 7562 dev_hold(fib_work->ven_info.dev); 7563 break; 7564 } 7565 } 7566 7567 static int mlxsw_sp_router_fib_rule_event(unsigned long event, 7568 struct fib_notifier_info *info, 7569 struct mlxsw_sp *mlxsw_sp) 7570 { 7571 struct netlink_ext_ack *extack = info->extack; 7572 struct fib_rule_notifier_info *fr_info; 7573 struct fib_rule *rule; 7574 int err = 0; 7575 7576 /* nothing to do at the moment */ 7577 if (event == FIB_EVENT_RULE_DEL) 7578 return 0; 7579 7580 fr_info = container_of(info, struct fib_rule_notifier_info, info); 7581 rule = fr_info->rule; 7582 7583 /* Rule only affects locally generated traffic */ 7584 if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex) 7585 return 0; 7586 7587 switch (info->family) { 7588 case AF_INET: 7589 if (!fib4_rule_default(rule) && !rule->l3mdev) 7590 err = -EOPNOTSUPP; 7591 break; 7592 case AF_INET6: 7593 if (!fib6_rule_default(rule) && !rule->l3mdev) 7594 err = -EOPNOTSUPP; 7595 break; 7596 case RTNL_FAMILY_IPMR: 7597 if (!ipmr_rule_default(rule) && !rule->l3mdev) 7598 err = -EOPNOTSUPP; 7599 break; 7600 case RTNL_FAMILY_IP6MR: 7601 if (!ip6mr_rule_default(rule) && !rule->l3mdev) 7602 err = -EOPNOTSUPP; 7603 break; 7604 } 7605 7606 if (err < 0) 7607 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported"); 7608 7609 return err; 7610 } 7611 7612 /* Called with rcu_read_lock() */ 7613 static int mlxsw_sp_router_fib_event(struct notifier_block *nb, 7614 unsigned long event, void *ptr) 7615 { 7616 struct mlxsw_sp_fib_event_work *fib_work; 7617 struct fib_notifier_info *info = ptr; 7618 struct mlxsw_sp_router *router; 7619 int err; 7620 7621 if ((info->family != AF_INET && info->family != AF_INET6 && 7622 info->family != RTNL_FAMILY_IPMR && 7623 info->family != RTNL_FAMILY_IP6MR)) 7624 return NOTIFY_DONE; 7625 7626 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 7627 7628 switch (event) { 7629 case FIB_EVENT_RULE_ADD: 7630 case FIB_EVENT_RULE_DEL: 7631 err = mlxsw_sp_router_fib_rule_event(event, info, 7632 router->mlxsw_sp); 7633 return notifier_from_errno(err); 7634 case FIB_EVENT_ENTRY_ADD: 7635 case FIB_EVENT_ENTRY_REPLACE: 7636 case FIB_EVENT_ENTRY_APPEND: 7637 if (info->family == AF_INET) { 7638 struct fib_entry_notifier_info *fen_info = ptr; 7639 7640 if (fen_info->fi->fib_nh_is_v6) { 7641 NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported"); 7642 return notifier_from_errno(-EINVAL); 7643 } 7644 } 7645 break; 7646 } 7647 7648 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC); 7649 if (!fib_work) 7650 return NOTIFY_BAD; 7651 7652 fib_work->mlxsw_sp = router->mlxsw_sp; 7653 fib_work->event = event; 7654 7655 switch (info->family) { 7656 case AF_INET: 7657 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work); 7658 mlxsw_sp_router_fib4_event(fib_work, info); 7659 break; 7660 case AF_INET6: 7661 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work); 7662 err = mlxsw_sp_router_fib6_event(fib_work, info); 7663 if (err) 7664 goto err_fib_event; 7665 break; 7666 case RTNL_FAMILY_IP6MR: 7667 case RTNL_FAMILY_IPMR: 7668 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work); 7669 mlxsw_sp_router_fibmr_event(fib_work, info); 7670 break; 7671 } 7672 7673 mlxsw_core_schedule_work(&fib_work->work); 7674 7675 return NOTIFY_DONE; 7676 7677 err_fib_event: 7678 kfree(fib_work); 7679 return NOTIFY_BAD; 7680 } 7681 7682 static struct mlxsw_sp_rif * 7683 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp, 7684 const struct net_device *dev) 7685 { 7686 int i; 7687 7688 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 7689 if (mlxsw_sp->router->rifs[i] && 7690 mlxsw_sp->router->rifs[i]->dev == dev) 7691 return mlxsw_sp->router->rifs[i]; 7692 7693 return NULL; 7694 } 7695 7696 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, 7697 const struct net_device *dev) 7698 { 7699 struct mlxsw_sp_rif *rif; 7700 7701 mutex_lock(&mlxsw_sp->router->lock); 7702 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7703 mutex_unlock(&mlxsw_sp->router->lock); 7704 7705 return rif; 7706 } 7707 7708 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev) 7709 { 7710 struct mlxsw_sp_rif *rif; 7711 u16 vid = 0; 7712 7713 mutex_lock(&mlxsw_sp->router->lock); 7714 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 7715 if (!rif) 7716 goto out; 7717 7718 /* We only return the VID for VLAN RIFs. Otherwise we return an 7719 * invalid value (0). 7720 */ 7721 if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN_EMU) 7722 goto out; 7723 7724 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 7725 7726 out: 7727 mutex_unlock(&mlxsw_sp->router->lock); 7728 return vid; 7729 } 7730 7731 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif) 7732 { 7733 char ritr_pl[MLXSW_REG_RITR_LEN]; 7734 int err; 7735 7736 mlxsw_reg_ritr_rif_pack(ritr_pl, rif); 7737 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7738 if (err) 7739 return err; 7740 7741 mlxsw_reg_ritr_enable_set(ritr_pl, false); 7742 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 7743 } 7744 7745 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp, 7746 struct mlxsw_sp_rif *rif) 7747 { 7748 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index); 7749 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif); 7750 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif); 7751 } 7752 7753 static bool 7754 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev, 7755 unsigned long event) 7756 { 7757 struct inet6_dev *inet6_dev; 7758 bool addr_list_empty = true; 7759 struct in_device *idev; 7760 7761 switch (event) { 7762 case NETDEV_UP: 7763 return rif == NULL; 7764 case NETDEV_DOWN: 7765 rcu_read_lock(); 7766 idev = __in_dev_get_rcu(dev); 7767 if (idev && idev->ifa_list) 7768 addr_list_empty = false; 7769 7770 inet6_dev = __in6_dev_get(dev); 7771 if (addr_list_empty && inet6_dev && 7772 !list_empty(&inet6_dev->addr_list)) 7773 addr_list_empty = false; 7774 rcu_read_unlock(); 7775 7776 /* macvlans do not have a RIF, but rather piggy back on the 7777 * RIF of their lower device. 7778 */ 7779 if (netif_is_macvlan(dev) && addr_list_empty) 7780 return true; 7781 7782 if (rif && addr_list_empty && 7783 !netif_is_l3_slave(rif->dev)) 7784 return true; 7785 /* It is possible we already removed the RIF ourselves 7786 * if it was assigned to a netdev that is now a bridge 7787 * or LAG slave. 7788 */ 7789 return false; 7790 } 7791 7792 return false; 7793 } 7794 7795 static enum mlxsw_sp_rif_type 7796 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp, 7797 const struct net_device *dev) 7798 { 7799 enum mlxsw_sp_fid_type type; 7800 7801 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL)) 7802 return MLXSW_SP_RIF_TYPE_IPIP_LB; 7803 7804 /* Otherwise RIF type is derived from the type of the underlying FID. */ 7805 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev))) 7806 type = MLXSW_SP_FID_TYPE_8021Q; 7807 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev)) 7808 type = MLXSW_SP_FID_TYPE_8021Q; 7809 else if (netif_is_bridge_master(dev)) 7810 type = MLXSW_SP_FID_TYPE_8021D; 7811 else 7812 type = MLXSW_SP_FID_TYPE_RFID; 7813 7814 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type); 7815 } 7816 7817 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index) 7818 { 7819 int i; 7820 7821 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) { 7822 if (!mlxsw_sp->router->rifs[i]) { 7823 *p_rif_index = i; 7824 return 0; 7825 } 7826 } 7827 7828 return -ENOBUFS; 7829 } 7830 7831 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index, 7832 u16 vr_id, 7833 struct net_device *l3_dev) 7834 { 7835 struct mlxsw_sp_rif *rif; 7836 7837 rif = kzalloc(rif_size, GFP_KERNEL); 7838 if (!rif) 7839 return NULL; 7840 7841 INIT_LIST_HEAD(&rif->nexthop_list); 7842 INIT_LIST_HEAD(&rif->neigh_list); 7843 if (l3_dev) { 7844 ether_addr_copy(rif->addr, l3_dev->dev_addr); 7845 rif->mtu = l3_dev->mtu; 7846 rif->dev = l3_dev; 7847 } 7848 rif->vr_id = vr_id; 7849 rif->rif_index = rif_index; 7850 7851 return rif; 7852 } 7853 7854 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp, 7855 u16 rif_index) 7856 { 7857 return mlxsw_sp->router->rifs[rif_index]; 7858 } 7859 7860 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif) 7861 { 7862 return rif->rif_index; 7863 } 7864 7865 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7866 { 7867 return lb_rif->common.rif_index; 7868 } 7869 7870 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7871 { 7872 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev); 7873 struct mlxsw_sp_vr *ul_vr; 7874 7875 ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL); 7876 if (WARN_ON(IS_ERR(ul_vr))) 7877 return 0; 7878 7879 return ul_vr->id; 7880 } 7881 7882 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif) 7883 { 7884 return lb_rif->ul_rif_id; 7885 } 7886 7887 static bool 7888 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif) 7889 { 7890 return mlxsw_sp_rif_counter_valid_get(rif, 7891 MLXSW_SP_RIF_COUNTER_EGRESS) && 7892 mlxsw_sp_rif_counter_valid_get(rif, 7893 MLXSW_SP_RIF_COUNTER_INGRESS); 7894 } 7895 7896 static int 7897 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif) 7898 { 7899 int err; 7900 7901 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 7902 if (err) 7903 return err; 7904 7905 /* Clear stale data. */ 7906 err = mlxsw_sp_rif_counter_fetch_clear(rif, 7907 MLXSW_SP_RIF_COUNTER_INGRESS, 7908 NULL); 7909 if (err) 7910 goto err_clear_ingress; 7911 7912 err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 7913 if (err) 7914 goto err_alloc_egress; 7915 7916 /* Clear stale data. */ 7917 err = mlxsw_sp_rif_counter_fetch_clear(rif, 7918 MLXSW_SP_RIF_COUNTER_EGRESS, 7919 NULL); 7920 if (err) 7921 goto err_clear_egress; 7922 7923 return 0; 7924 7925 err_clear_egress: 7926 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 7927 err_alloc_egress: 7928 err_clear_ingress: 7929 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 7930 return err; 7931 } 7932 7933 static void 7934 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif) 7935 { 7936 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS); 7937 mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS); 7938 } 7939 7940 static void 7941 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif, 7942 struct netdev_notifier_offload_xstats_info *info) 7943 { 7944 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 7945 return; 7946 netdev_offload_xstats_report_used(info->report_used); 7947 } 7948 7949 static int 7950 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif, 7951 struct rtnl_hw_stats64 *p_stats) 7952 { 7953 struct mlxsw_sp_rif_counter_set_basic ingress; 7954 struct mlxsw_sp_rif_counter_set_basic egress; 7955 int err; 7956 7957 err = mlxsw_sp_rif_counter_fetch_clear(rif, 7958 MLXSW_SP_RIF_COUNTER_INGRESS, 7959 &ingress); 7960 if (err) 7961 return err; 7962 7963 err = mlxsw_sp_rif_counter_fetch_clear(rif, 7964 MLXSW_SP_RIF_COUNTER_EGRESS, 7965 &egress); 7966 if (err) 7967 return err; 7968 7969 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX) \ 7970 ((SET.good_unicast_ ## SFX) + \ 7971 (SET.good_multicast_ ## SFX) + \ 7972 (SET.good_broadcast_ ## SFX)) 7973 7974 p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets); 7975 p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets); 7976 p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes); 7977 p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes); 7978 p_stats->rx_errors = ingress.error_packets; 7979 p_stats->tx_errors = egress.error_packets; 7980 p_stats->rx_dropped = ingress.discard_packets; 7981 p_stats->tx_dropped = egress.discard_packets; 7982 p_stats->multicast = ingress.good_multicast_packets + 7983 ingress.good_broadcast_packets; 7984 7985 #undef MLXSW_SP_ROUTER_ALL_GOOD 7986 7987 return 0; 7988 } 7989 7990 static int 7991 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif, 7992 struct netdev_notifier_offload_xstats_info *info) 7993 { 7994 struct rtnl_hw_stats64 stats = {}; 7995 int err; 7996 7997 if (!mlxsw_sp_router_port_l3_stats_enabled(rif)) 7998 return 0; 7999 8000 err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats); 8001 if (err) 8002 return err; 8003 8004 netdev_offload_xstats_report_delta(info->report_delta, &stats); 8005 return 0; 8006 } 8007 8008 struct mlxsw_sp_router_hwstats_notify_work { 8009 struct work_struct work; 8010 struct net_device *dev; 8011 }; 8012 8013 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work) 8014 { 8015 struct mlxsw_sp_router_hwstats_notify_work *hws_work = 8016 container_of(work, struct mlxsw_sp_router_hwstats_notify_work, 8017 work); 8018 8019 rtnl_lock(); 8020 rtnl_offload_xstats_notify(hws_work->dev); 8021 rtnl_unlock(); 8022 dev_put(hws_work->dev); 8023 kfree(hws_work); 8024 } 8025 8026 static void 8027 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev) 8028 { 8029 struct mlxsw_sp_router_hwstats_notify_work *hws_work; 8030 8031 /* To collect notification payload, the core ends up sending another 8032 * notifier block message, which would deadlock on the attempt to 8033 * acquire the router lock again. Just postpone the notification until 8034 * later. 8035 */ 8036 8037 hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL); 8038 if (!hws_work) 8039 return; 8040 8041 INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work); 8042 dev_hold(dev); 8043 hws_work->dev = dev; 8044 mlxsw_core_schedule_work(&hws_work->work); 8045 } 8046 8047 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif) 8048 { 8049 return rif->dev->ifindex; 8050 } 8051 8052 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif) 8053 { 8054 return rif->dev; 8055 } 8056 8057 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif) 8058 { 8059 struct rtnl_hw_stats64 stats = {}; 8060 8061 if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats)) 8062 netdev_offload_xstats_push_delta(rif->dev, 8063 NETDEV_OFFLOAD_XSTATS_TYPE_L3, 8064 &stats); 8065 } 8066 8067 static struct mlxsw_sp_rif * 8068 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp, 8069 const struct mlxsw_sp_rif_params *params, 8070 struct netlink_ext_ack *extack) 8071 { 8072 u32 tb_id = l3mdev_fib_table(params->dev); 8073 const struct mlxsw_sp_rif_ops *ops; 8074 struct mlxsw_sp_fid *fid = NULL; 8075 enum mlxsw_sp_rif_type type; 8076 struct mlxsw_sp_rif *rif; 8077 struct mlxsw_sp_vr *vr; 8078 u16 rif_index; 8079 int i, err; 8080 8081 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev); 8082 ops = mlxsw_sp->router->rif_ops_arr[type]; 8083 8084 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack); 8085 if (IS_ERR(vr)) 8086 return ERR_CAST(vr); 8087 vr->rif_count++; 8088 8089 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 8090 if (err) { 8091 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 8092 goto err_rif_index_alloc; 8093 } 8094 8095 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev); 8096 if (!rif) { 8097 err = -ENOMEM; 8098 goto err_rif_alloc; 8099 } 8100 dev_hold(rif->dev); 8101 mlxsw_sp->router->rifs[rif_index] = rif; 8102 rif->mlxsw_sp = mlxsw_sp; 8103 rif->ops = ops; 8104 8105 if (ops->fid_get) { 8106 fid = ops->fid_get(rif, extack); 8107 if (IS_ERR(fid)) { 8108 err = PTR_ERR(fid); 8109 goto err_fid_get; 8110 } 8111 rif->fid = fid; 8112 } 8113 8114 if (ops->setup) 8115 ops->setup(rif, params); 8116 8117 err = ops->configure(rif, extack); 8118 if (err) 8119 goto err_configure; 8120 8121 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) { 8122 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif); 8123 if (err) 8124 goto err_mr_rif_add; 8125 } 8126 8127 if (netdev_offload_xstats_enabled(rif->dev, 8128 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8129 err = mlxsw_sp_router_port_l3_stats_enable(rif); 8130 if (err) 8131 goto err_stats_enable; 8132 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8133 } else { 8134 mlxsw_sp_rif_counters_alloc(rif); 8135 } 8136 8137 atomic_inc(&mlxsw_sp->router->rifs_count); 8138 return rif; 8139 8140 err_stats_enable: 8141 err_mr_rif_add: 8142 for (i--; i >= 0; i--) 8143 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8144 ops->deconfigure(rif); 8145 err_configure: 8146 if (fid) 8147 mlxsw_sp_fid_put(fid); 8148 err_fid_get: 8149 mlxsw_sp->router->rifs[rif_index] = NULL; 8150 dev_put(rif->dev); 8151 kfree(rif); 8152 err_rif_alloc: 8153 err_rif_index_alloc: 8154 vr->rif_count--; 8155 mlxsw_sp_vr_put(mlxsw_sp, vr); 8156 return ERR_PTR(err); 8157 } 8158 8159 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif) 8160 { 8161 const struct mlxsw_sp_rif_ops *ops = rif->ops; 8162 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8163 struct mlxsw_sp_fid *fid = rif->fid; 8164 struct mlxsw_sp_vr *vr; 8165 int i; 8166 8167 atomic_dec(&mlxsw_sp->router->rifs_count); 8168 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif); 8169 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 8170 8171 if (netdev_offload_xstats_enabled(rif->dev, 8172 NETDEV_OFFLOAD_XSTATS_TYPE_L3)) { 8173 mlxsw_sp_rif_push_l3_stats(rif); 8174 mlxsw_sp_router_port_l3_stats_disable(rif); 8175 mlxsw_sp_router_hwstats_notify_schedule(rif->dev); 8176 } else { 8177 mlxsw_sp_rif_counters_free(rif); 8178 } 8179 8180 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 8181 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif); 8182 ops->deconfigure(rif); 8183 if (fid) 8184 /* Loopback RIFs are not associated with a FID. */ 8185 mlxsw_sp_fid_put(fid); 8186 mlxsw_sp->router->rifs[rif->rif_index] = NULL; 8187 dev_put(rif->dev); 8188 kfree(rif); 8189 vr->rif_count--; 8190 mlxsw_sp_vr_put(mlxsw_sp, vr); 8191 } 8192 8193 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, 8194 struct net_device *dev) 8195 { 8196 struct mlxsw_sp_rif *rif; 8197 8198 mutex_lock(&mlxsw_sp->router->lock); 8199 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8200 if (!rif) 8201 goto out; 8202 mlxsw_sp_rif_destroy(rif); 8203 out: 8204 mutex_unlock(&mlxsw_sp->router->lock); 8205 } 8206 8207 static void 8208 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params, 8209 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8210 { 8211 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8212 8213 params->vid = mlxsw_sp_port_vlan->vid; 8214 params->lag = mlxsw_sp_port->lagged; 8215 if (params->lag) 8216 params->lag_id = mlxsw_sp_port->lag_id; 8217 else 8218 params->system_port = mlxsw_sp_port->local_port; 8219 } 8220 8221 static struct mlxsw_sp_rif_subport * 8222 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif) 8223 { 8224 return container_of(rif, struct mlxsw_sp_rif_subport, common); 8225 } 8226 8227 static struct mlxsw_sp_rif * 8228 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp, 8229 const struct mlxsw_sp_rif_params *params, 8230 struct netlink_ext_ack *extack) 8231 { 8232 struct mlxsw_sp_rif_subport *rif_subport; 8233 struct mlxsw_sp_rif *rif; 8234 8235 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev); 8236 if (!rif) 8237 return mlxsw_sp_rif_create(mlxsw_sp, params, extack); 8238 8239 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8240 refcount_inc(&rif_subport->ref_count); 8241 return rif; 8242 } 8243 8244 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif) 8245 { 8246 struct mlxsw_sp_rif_subport *rif_subport; 8247 8248 rif_subport = mlxsw_sp_rif_subport_rif(rif); 8249 if (!refcount_dec_and_test(&rif_subport->ref_count)) 8250 return; 8251 8252 mlxsw_sp_rif_destroy(rif); 8253 } 8254 8255 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp, 8256 struct mlxsw_sp_rif_mac_profile *profile, 8257 struct netlink_ext_ack *extack) 8258 { 8259 u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 8260 struct mlxsw_sp_router *router = mlxsw_sp->router; 8261 int id; 8262 8263 id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0, 8264 max_rif_mac_profiles, GFP_KERNEL); 8265 8266 if (id >= 0) { 8267 profile->id = id; 8268 return 0; 8269 } 8270 8271 if (id == -ENOSPC) 8272 NL_SET_ERR_MSG_MOD(extack, 8273 "Exceeded number of supported router interface MAC profiles"); 8274 8275 return id; 8276 } 8277 8278 static struct mlxsw_sp_rif_mac_profile * 8279 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile) 8280 { 8281 struct mlxsw_sp_rif_mac_profile *profile; 8282 8283 profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr, 8284 mac_profile); 8285 WARN_ON(!profile); 8286 return profile; 8287 } 8288 8289 static struct mlxsw_sp_rif_mac_profile * 8290 mlxsw_sp_rif_mac_profile_alloc(const char *mac) 8291 { 8292 struct mlxsw_sp_rif_mac_profile *profile; 8293 8294 profile = kzalloc(sizeof(*profile), GFP_KERNEL); 8295 if (!profile) 8296 return NULL; 8297 8298 ether_addr_copy(profile->mac_prefix, mac); 8299 refcount_set(&profile->ref_count, 1); 8300 return profile; 8301 } 8302 8303 static struct mlxsw_sp_rif_mac_profile * 8304 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) 8305 { 8306 struct mlxsw_sp_router *router = mlxsw_sp->router; 8307 struct mlxsw_sp_rif_mac_profile *profile; 8308 int id; 8309 8310 idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { 8311 if (ether_addr_equal_masked(profile->mac_prefix, mac, 8312 mlxsw_sp->mac_mask)) 8313 return profile; 8314 } 8315 8316 return NULL; 8317 } 8318 8319 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv) 8320 { 8321 const struct mlxsw_sp *mlxsw_sp = priv; 8322 8323 return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count); 8324 } 8325 8326 static u64 mlxsw_sp_rifs_occ_get(void *priv) 8327 { 8328 const struct mlxsw_sp *mlxsw_sp = priv; 8329 8330 return atomic_read(&mlxsw_sp->router->rifs_count); 8331 } 8332 8333 static struct mlxsw_sp_rif_mac_profile * 8334 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac, 8335 struct netlink_ext_ack *extack) 8336 { 8337 struct mlxsw_sp_rif_mac_profile *profile; 8338 int err; 8339 8340 profile = mlxsw_sp_rif_mac_profile_alloc(mac); 8341 if (!profile) 8342 return ERR_PTR(-ENOMEM); 8343 8344 err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack); 8345 if (err) 8346 goto profile_index_alloc_err; 8347 8348 atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count); 8349 return profile; 8350 8351 profile_index_alloc_err: 8352 kfree(profile); 8353 return ERR_PTR(err); 8354 } 8355 8356 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp, 8357 u8 mac_profile) 8358 { 8359 struct mlxsw_sp_rif_mac_profile *profile; 8360 8361 atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count); 8362 profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile); 8363 kfree(profile); 8364 } 8365 8366 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp, 8367 const char *mac, u8 *p_mac_profile, 8368 struct netlink_ext_ack *extack) 8369 { 8370 struct mlxsw_sp_rif_mac_profile *profile; 8371 8372 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac); 8373 if (profile) { 8374 refcount_inc(&profile->ref_count); 8375 goto out; 8376 } 8377 8378 profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack); 8379 if (IS_ERR(profile)) 8380 return PTR_ERR(profile); 8381 8382 out: 8383 *p_mac_profile = profile->id; 8384 return 0; 8385 } 8386 8387 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp, 8388 u8 mac_profile) 8389 { 8390 struct mlxsw_sp_rif_mac_profile *profile; 8391 8392 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8393 mac_profile); 8394 if (WARN_ON(!profile)) 8395 return; 8396 8397 if (!refcount_dec_and_test(&profile->ref_count)) 8398 return; 8399 8400 mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile); 8401 } 8402 8403 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif) 8404 { 8405 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8406 struct mlxsw_sp_rif_mac_profile *profile; 8407 8408 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8409 rif->mac_profile_id); 8410 if (WARN_ON(!profile)) 8411 return false; 8412 8413 return refcount_read(&profile->ref_count) > 1; 8414 } 8415 8416 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif, 8417 const char *new_mac) 8418 { 8419 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 8420 struct mlxsw_sp_rif_mac_profile *profile; 8421 8422 profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr, 8423 rif->mac_profile_id); 8424 if (WARN_ON(!profile)) 8425 return -EINVAL; 8426 8427 ether_addr_copy(profile->mac_prefix, new_mac); 8428 return 0; 8429 } 8430 8431 static int 8432 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp, 8433 struct mlxsw_sp_rif *rif, 8434 const char *new_mac, 8435 struct netlink_ext_ack *extack) 8436 { 8437 u8 mac_profile; 8438 int err; 8439 8440 if (!mlxsw_sp_rif_mac_profile_is_shared(rif) && 8441 !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac)) 8442 return mlxsw_sp_rif_mac_profile_edit(rif, new_mac); 8443 8444 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac, 8445 &mac_profile, extack); 8446 if (err) 8447 return err; 8448 8449 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id); 8450 rif->mac_profile_id = mac_profile; 8451 return 0; 8452 } 8453 8454 static int 8455 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8456 struct net_device *l3_dev, 8457 struct netlink_ext_ack *extack) 8458 { 8459 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8460 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 8461 struct mlxsw_sp_rif_params params = { 8462 .dev = l3_dev, 8463 }; 8464 u16 vid = mlxsw_sp_port_vlan->vid; 8465 struct mlxsw_sp_rif *rif; 8466 struct mlxsw_sp_fid *fid; 8467 int err; 8468 8469 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan); 8470 rif = mlxsw_sp_rif_subport_get(mlxsw_sp, ¶ms, extack); 8471 if (IS_ERR(rif)) 8472 return PTR_ERR(rif); 8473 8474 /* FID was already created, just take a reference */ 8475 fid = rif->ops->fid_get(rif, extack); 8476 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid); 8477 if (err) 8478 goto err_fid_port_vid_map; 8479 8480 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false); 8481 if (err) 8482 goto err_port_vid_learning_set; 8483 8484 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, 8485 BR_STATE_FORWARDING); 8486 if (err) 8487 goto err_port_vid_stp_set; 8488 8489 mlxsw_sp_port_vlan->fid = fid; 8490 8491 return 0; 8492 8493 err_port_vid_stp_set: 8494 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8495 err_port_vid_learning_set: 8496 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8497 err_fid_port_vid_map: 8498 mlxsw_sp_fid_put(fid); 8499 mlxsw_sp_rif_subport_put(rif); 8500 return err; 8501 } 8502 8503 static void 8504 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8505 { 8506 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; 8507 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; 8508 struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid); 8509 u16 vid = mlxsw_sp_port_vlan->vid; 8510 8511 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID)) 8512 return; 8513 8514 mlxsw_sp_port_vlan->fid = NULL; 8515 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING); 8516 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true); 8517 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid); 8518 mlxsw_sp_fid_put(fid); 8519 mlxsw_sp_rif_subport_put(rif); 8520 } 8521 8522 int 8523 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, 8524 struct net_device *l3_dev, 8525 struct netlink_ext_ack *extack) 8526 { 8527 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8528 struct mlxsw_sp_rif *rif; 8529 int err = 0; 8530 8531 mutex_lock(&mlxsw_sp->router->lock); 8532 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8533 if (!rif) 8534 goto out; 8535 8536 err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev, 8537 extack); 8538 out: 8539 mutex_unlock(&mlxsw_sp->router->lock); 8540 return err; 8541 } 8542 8543 void 8544 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) 8545 { 8546 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp; 8547 8548 mutex_lock(&mlxsw_sp->router->lock); 8549 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8550 mutex_unlock(&mlxsw_sp->router->lock); 8551 } 8552 8553 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev, 8554 struct net_device *port_dev, 8555 unsigned long event, u16 vid, 8556 struct netlink_ext_ack *extack) 8557 { 8558 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev); 8559 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan; 8560 8561 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid); 8562 if (WARN_ON(!mlxsw_sp_port_vlan)) 8563 return -EINVAL; 8564 8565 switch (event) { 8566 case NETDEV_UP: 8567 return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, 8568 l3_dev, extack); 8569 case NETDEV_DOWN: 8570 __mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan); 8571 break; 8572 } 8573 8574 return 0; 8575 } 8576 8577 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev, 8578 unsigned long event, 8579 struct netlink_ext_ack *extack) 8580 { 8581 if (netif_is_bridge_port(port_dev) || 8582 netif_is_lag_port(port_dev) || 8583 netif_is_ovs_port(port_dev)) 8584 return 0; 8585 8586 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 8587 MLXSW_SP_DEFAULT_VID, extack); 8588 } 8589 8590 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev, 8591 struct net_device *lag_dev, 8592 unsigned long event, u16 vid, 8593 struct netlink_ext_ack *extack) 8594 { 8595 struct net_device *port_dev; 8596 struct list_head *iter; 8597 int err; 8598 8599 netdev_for_each_lower_dev(lag_dev, port_dev, iter) { 8600 if (mlxsw_sp_port_dev_check(port_dev)) { 8601 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev, 8602 port_dev, 8603 event, vid, 8604 extack); 8605 if (err) 8606 return err; 8607 } 8608 } 8609 8610 return 0; 8611 } 8612 8613 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev, 8614 unsigned long event, 8615 struct netlink_ext_ack *extack) 8616 { 8617 if (netif_is_bridge_port(lag_dev)) 8618 return 0; 8619 8620 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 8621 MLXSW_SP_DEFAULT_VID, extack); 8622 } 8623 8624 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp, 8625 struct net_device *l3_dev, 8626 unsigned long event, 8627 struct netlink_ext_ack *extack) 8628 { 8629 struct mlxsw_sp_rif_params params = { 8630 .dev = l3_dev, 8631 }; 8632 struct mlxsw_sp_rif *rif; 8633 8634 switch (event) { 8635 case NETDEV_UP: 8636 if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) { 8637 u16 proto; 8638 8639 br_vlan_get_proto(l3_dev, &proto); 8640 if (proto == ETH_P_8021AD) { 8641 NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported"); 8642 return -EOPNOTSUPP; 8643 } 8644 } 8645 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack); 8646 if (IS_ERR(rif)) 8647 return PTR_ERR(rif); 8648 break; 8649 case NETDEV_DOWN: 8650 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 8651 mlxsw_sp_rif_destroy(rif); 8652 break; 8653 } 8654 8655 return 0; 8656 } 8657 8658 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp, 8659 struct net_device *vlan_dev, 8660 unsigned long event, 8661 struct netlink_ext_ack *extack) 8662 { 8663 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev); 8664 u16 vid = vlan_dev_vlan_id(vlan_dev); 8665 8666 if (netif_is_bridge_port(vlan_dev)) 8667 return 0; 8668 8669 if (mlxsw_sp_port_dev_check(real_dev)) 8670 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev, 8671 event, vid, extack); 8672 else if (netif_is_lag_master(real_dev)) 8673 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event, 8674 vid, extack); 8675 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev)) 8676 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event, 8677 extack); 8678 8679 return 0; 8680 } 8681 8682 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac) 8683 { 8684 u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 }; 8685 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8686 8687 return ether_addr_equal_masked(mac, vrrp4, mask); 8688 } 8689 8690 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac) 8691 { 8692 u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 }; 8693 u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; 8694 8695 return ether_addr_equal_masked(mac, vrrp6, mask); 8696 } 8697 8698 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8699 const u8 *mac, bool adding) 8700 { 8701 char ritr_pl[MLXSW_REG_RITR_LEN]; 8702 u8 vrrp_id = adding ? mac[5] : 0; 8703 int err; 8704 8705 if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) && 8706 !mlxsw_sp_rif_macvlan_is_vrrp6(mac)) 8707 return 0; 8708 8709 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8710 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8711 if (err) 8712 return err; 8713 8714 if (mlxsw_sp_rif_macvlan_is_vrrp4(mac)) 8715 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id); 8716 else 8717 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id); 8718 8719 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8720 } 8721 8722 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp, 8723 const struct net_device *macvlan_dev, 8724 struct netlink_ext_ack *extack) 8725 { 8726 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8727 struct mlxsw_sp_rif *rif; 8728 int err; 8729 8730 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8731 if (!rif) { 8732 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces"); 8733 return -EOPNOTSUPP; 8734 } 8735 8736 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8737 mlxsw_sp_fid_index(rif->fid), true); 8738 if (err) 8739 return err; 8740 8741 err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, 8742 macvlan_dev->dev_addr, true); 8743 if (err) 8744 goto err_rif_vrrp_add; 8745 8746 /* Make sure the bridge driver does not have this MAC pointing at 8747 * some other port. 8748 */ 8749 if (rif->ops->fdb_del) 8750 rif->ops->fdb_del(rif, macvlan_dev->dev_addr); 8751 8752 return 0; 8753 8754 err_rif_vrrp_add: 8755 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8756 mlxsw_sp_fid_index(rif->fid), false); 8757 return err; 8758 } 8759 8760 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8761 const struct net_device *macvlan_dev) 8762 { 8763 struct macvlan_dev *vlan = netdev_priv(macvlan_dev); 8764 struct mlxsw_sp_rif *rif; 8765 8766 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev); 8767 /* If we do not have a RIF, then we already took care of 8768 * removing the macvlan's MAC during RIF deletion. 8769 */ 8770 if (!rif) 8771 return; 8772 mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr, 8773 false); 8774 mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr, 8775 mlxsw_sp_fid_index(rif->fid), false); 8776 } 8777 8778 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp, 8779 const struct net_device *macvlan_dev) 8780 { 8781 mutex_lock(&mlxsw_sp->router->lock); 8782 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8783 mutex_unlock(&mlxsw_sp->router->lock); 8784 } 8785 8786 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp, 8787 struct net_device *macvlan_dev, 8788 unsigned long event, 8789 struct netlink_ext_ack *extack) 8790 { 8791 switch (event) { 8792 case NETDEV_UP: 8793 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack); 8794 case NETDEV_DOWN: 8795 __mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev); 8796 break; 8797 } 8798 8799 return 0; 8800 } 8801 8802 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp, 8803 struct net_device *dev, 8804 unsigned long event, 8805 struct netlink_ext_ack *extack) 8806 { 8807 if (mlxsw_sp_port_dev_check(dev)) 8808 return mlxsw_sp_inetaddr_port_event(dev, event, extack); 8809 else if (netif_is_lag_master(dev)) 8810 return mlxsw_sp_inetaddr_lag_event(dev, event, extack); 8811 else if (netif_is_bridge_master(dev)) 8812 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event, 8813 extack); 8814 else if (is_vlan_dev(dev)) 8815 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event, 8816 extack); 8817 else if (netif_is_macvlan(dev)) 8818 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event, 8819 extack); 8820 else 8821 return 0; 8822 } 8823 8824 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb, 8825 unsigned long event, void *ptr) 8826 { 8827 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr; 8828 struct net_device *dev = ifa->ifa_dev->dev; 8829 struct mlxsw_sp_router *router; 8830 struct mlxsw_sp_rif *rif; 8831 int err = 0; 8832 8833 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */ 8834 if (event == NETDEV_UP) 8835 return NOTIFY_DONE; 8836 8837 router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb); 8838 mutex_lock(&router->lock); 8839 rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev); 8840 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8841 goto out; 8842 8843 err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL); 8844 out: 8845 mutex_unlock(&router->lock); 8846 return notifier_from_errno(err); 8847 } 8848 8849 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused, 8850 unsigned long event, void *ptr) 8851 { 8852 struct in_validator_info *ivi = (struct in_validator_info *) ptr; 8853 struct net_device *dev = ivi->ivi_dev->dev; 8854 struct mlxsw_sp *mlxsw_sp; 8855 struct mlxsw_sp_rif *rif; 8856 int err = 0; 8857 8858 mlxsw_sp = mlxsw_sp_lower_get(dev); 8859 if (!mlxsw_sp) 8860 return NOTIFY_DONE; 8861 8862 mutex_lock(&mlxsw_sp->router->lock); 8863 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8864 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8865 goto out; 8866 8867 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack); 8868 out: 8869 mutex_unlock(&mlxsw_sp->router->lock); 8870 return notifier_from_errno(err); 8871 } 8872 8873 struct mlxsw_sp_inet6addr_event_work { 8874 struct work_struct work; 8875 struct mlxsw_sp *mlxsw_sp; 8876 struct net_device *dev; 8877 unsigned long event; 8878 }; 8879 8880 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work) 8881 { 8882 struct mlxsw_sp_inet6addr_event_work *inet6addr_work = 8883 container_of(work, struct mlxsw_sp_inet6addr_event_work, work); 8884 struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp; 8885 struct net_device *dev = inet6addr_work->dev; 8886 unsigned long event = inet6addr_work->event; 8887 struct mlxsw_sp_rif *rif; 8888 8889 rtnl_lock(); 8890 mutex_lock(&mlxsw_sp->router->lock); 8891 8892 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8893 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8894 goto out; 8895 8896 __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL); 8897 out: 8898 mutex_unlock(&mlxsw_sp->router->lock); 8899 rtnl_unlock(); 8900 dev_put(dev); 8901 kfree(inet6addr_work); 8902 } 8903 8904 /* Called with rcu_read_lock() */ 8905 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb, 8906 unsigned long event, void *ptr) 8907 { 8908 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr; 8909 struct mlxsw_sp_inet6addr_event_work *inet6addr_work; 8910 struct net_device *dev = if6->idev->dev; 8911 struct mlxsw_sp_router *router; 8912 8913 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */ 8914 if (event == NETDEV_UP) 8915 return NOTIFY_DONE; 8916 8917 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC); 8918 if (!inet6addr_work) 8919 return NOTIFY_BAD; 8920 8921 router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb); 8922 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work); 8923 inet6addr_work->mlxsw_sp = router->mlxsw_sp; 8924 inet6addr_work->dev = dev; 8925 inet6addr_work->event = event; 8926 dev_hold(dev); 8927 mlxsw_core_schedule_work(&inet6addr_work->work); 8928 8929 return NOTIFY_DONE; 8930 } 8931 8932 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused, 8933 unsigned long event, void *ptr) 8934 { 8935 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr; 8936 struct net_device *dev = i6vi->i6vi_dev->dev; 8937 struct mlxsw_sp *mlxsw_sp; 8938 struct mlxsw_sp_rif *rif; 8939 int err = 0; 8940 8941 mlxsw_sp = mlxsw_sp_lower_get(dev); 8942 if (!mlxsw_sp) 8943 return NOTIFY_DONE; 8944 8945 mutex_lock(&mlxsw_sp->router->lock); 8946 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 8947 if (!mlxsw_sp_rif_should_config(rif, dev, event)) 8948 goto out; 8949 8950 err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack); 8951 out: 8952 mutex_unlock(&mlxsw_sp->router->lock); 8953 return notifier_from_errno(err); 8954 } 8955 8956 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index, 8957 const char *mac, int mtu, u8 mac_profile) 8958 { 8959 char ritr_pl[MLXSW_REG_RITR_LEN]; 8960 int err; 8961 8962 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index); 8963 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8964 if (err) 8965 return err; 8966 8967 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu); 8968 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac); 8969 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile); 8970 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE); 8971 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 8972 } 8973 8974 static int 8975 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp, 8976 struct mlxsw_sp_rif *rif, 8977 struct netlink_ext_ack *extack) 8978 { 8979 struct net_device *dev = rif->dev; 8980 u8 old_mac_profile; 8981 u16 fid_index; 8982 int err; 8983 8984 fid_index = mlxsw_sp_fid_index(rif->fid); 8985 8986 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false); 8987 if (err) 8988 return err; 8989 8990 old_mac_profile = rif->mac_profile_id; 8991 err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr, 8992 extack); 8993 if (err) 8994 goto err_rif_mac_profile_replace; 8995 8996 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr, 8997 dev->mtu, rif->mac_profile_id); 8998 if (err) 8999 goto err_rif_edit; 9000 9001 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true); 9002 if (err) 9003 goto err_rif_fdb_op; 9004 9005 if (rif->mtu != dev->mtu) { 9006 struct mlxsw_sp_vr *vr; 9007 int i; 9008 9009 /* The RIF is relevant only to its mr_table instance, as unlike 9010 * unicast routing, in multicast routing a RIF cannot be shared 9011 * between several multicast routing tables. 9012 */ 9013 vr = &mlxsw_sp->router->vrs[rif->vr_id]; 9014 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) 9015 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i], 9016 rif, dev->mtu); 9017 } 9018 9019 ether_addr_copy(rif->addr, dev->dev_addr); 9020 rif->mtu = dev->mtu; 9021 9022 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index); 9023 9024 return 0; 9025 9026 err_rif_fdb_op: 9027 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu, 9028 old_mac_profile); 9029 err_rif_edit: 9030 mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack); 9031 err_rif_mac_profile_replace: 9032 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true); 9033 return err; 9034 } 9035 9036 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif, 9037 struct netdev_notifier_pre_changeaddr_info *info) 9038 { 9039 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9040 struct mlxsw_sp_rif_mac_profile *profile; 9041 struct netlink_ext_ack *extack; 9042 u8 max_rif_mac_profiles; 9043 u64 occ; 9044 9045 extack = netdev_notifier_info_to_extack(&info->info); 9046 9047 profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr); 9048 if (profile) 9049 return 0; 9050 9051 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile; 9052 occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp); 9053 if (occ < max_rif_mac_profiles) 9054 return 0; 9055 9056 if (!mlxsw_sp_rif_mac_profile_is_shared(rif)) 9057 return 0; 9058 9059 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles"); 9060 return -ENOBUFS; 9061 } 9062 9063 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event) 9064 { 9065 switch (event) { 9066 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9067 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9068 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9069 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9070 return true; 9071 } 9072 9073 return false; 9074 } 9075 9076 static int 9077 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif, 9078 unsigned long event, 9079 struct netdev_notifier_offload_xstats_info *info) 9080 { 9081 switch (info->type) { 9082 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9083 break; 9084 default: 9085 return 0; 9086 } 9087 9088 switch (event) { 9089 case NETDEV_OFFLOAD_XSTATS_ENABLE: 9090 return mlxsw_sp_router_port_l3_stats_enable(rif); 9091 case NETDEV_OFFLOAD_XSTATS_DISABLE: 9092 mlxsw_sp_router_port_l3_stats_disable(rif); 9093 return 0; 9094 case NETDEV_OFFLOAD_XSTATS_REPORT_USED: 9095 mlxsw_sp_router_port_l3_stats_report_used(rif, info); 9096 return 0; 9097 case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA: 9098 return mlxsw_sp_router_port_l3_stats_report_delta(rif, info); 9099 } 9100 9101 WARN_ON_ONCE(1); 9102 return 0; 9103 } 9104 9105 static int 9106 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp, 9107 struct net_device *dev, 9108 unsigned long event, 9109 struct netdev_notifier_offload_xstats_info *info) 9110 { 9111 struct mlxsw_sp_rif *rif; 9112 9113 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9114 if (!rif) 9115 return 0; 9116 9117 return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info); 9118 } 9119 9120 static bool mlxsw_sp_is_router_event(unsigned long event) 9121 { 9122 switch (event) { 9123 case NETDEV_PRE_CHANGEADDR: 9124 case NETDEV_CHANGEADDR: 9125 case NETDEV_CHANGEMTU: 9126 return true; 9127 default: 9128 return false; 9129 } 9130 } 9131 9132 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev, 9133 unsigned long event, void *ptr) 9134 { 9135 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr); 9136 struct mlxsw_sp *mlxsw_sp; 9137 struct mlxsw_sp_rif *rif; 9138 9139 mlxsw_sp = mlxsw_sp_lower_get(dev); 9140 if (!mlxsw_sp) 9141 return 0; 9142 9143 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev); 9144 if (!rif) 9145 return 0; 9146 9147 switch (event) { 9148 case NETDEV_CHANGEMTU: 9149 case NETDEV_CHANGEADDR: 9150 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack); 9151 case NETDEV_PRE_CHANGEADDR: 9152 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr); 9153 default: 9154 WARN_ON_ONCE(1); 9155 break; 9156 } 9157 9158 return 0; 9159 } 9160 9161 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp, 9162 struct net_device *l3_dev, 9163 struct netlink_ext_ack *extack) 9164 { 9165 struct mlxsw_sp_rif *rif; 9166 9167 /* If netdev is already associated with a RIF, then we need to 9168 * destroy it and create a new one with the new virtual router ID. 9169 */ 9170 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9171 if (rif) 9172 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, 9173 extack); 9174 9175 return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack); 9176 } 9177 9178 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp, 9179 struct net_device *l3_dev) 9180 { 9181 struct mlxsw_sp_rif *rif; 9182 9183 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev); 9184 if (!rif) 9185 return; 9186 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL); 9187 } 9188 9189 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr) 9190 { 9191 struct netdev_notifier_changeupper_info *info = ptr; 9192 9193 if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER) 9194 return false; 9195 return netif_is_l3_master(info->upper_dev); 9196 } 9197 9198 static int 9199 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event, 9200 struct netdev_notifier_changeupper_info *info) 9201 { 9202 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev); 9203 int err = 0; 9204 9205 /* We do not create a RIF for a macvlan, but only use it to 9206 * direct more MAC addresses to the router. 9207 */ 9208 if (!mlxsw_sp || netif_is_macvlan(l3_dev)) 9209 return 0; 9210 9211 switch (event) { 9212 case NETDEV_PRECHANGEUPPER: 9213 break; 9214 case NETDEV_CHANGEUPPER: 9215 if (info->linking) { 9216 struct netlink_ext_ack *extack; 9217 9218 extack = netdev_notifier_info_to_extack(&info->info); 9219 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack); 9220 } else { 9221 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev); 9222 } 9223 break; 9224 } 9225 9226 return err; 9227 } 9228 9229 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb, 9230 unsigned long event, void *ptr) 9231 { 9232 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 9233 struct mlxsw_sp_router *router; 9234 struct mlxsw_sp *mlxsw_sp; 9235 int err = 0; 9236 9237 router = container_of(nb, struct mlxsw_sp_router, netdevice_nb); 9238 mlxsw_sp = router->mlxsw_sp; 9239 9240 mutex_lock(&mlxsw_sp->router->lock); 9241 9242 if (mlxsw_sp_is_offload_xstats_event(event)) 9243 err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev, 9244 event, ptr); 9245 else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev)) 9246 err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev, 9247 event, ptr); 9248 else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev)) 9249 err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev, 9250 event, ptr); 9251 else if (mlxsw_sp_is_router_event(event)) 9252 err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr); 9253 else if (mlxsw_sp_is_vrf_event(event, ptr)) 9254 err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr); 9255 9256 mutex_unlock(&mlxsw_sp->router->lock); 9257 9258 return notifier_from_errno(err); 9259 } 9260 9261 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, 9262 struct netdev_nested_priv *priv) 9263 { 9264 struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data; 9265 9266 if (!netif_is_macvlan(dev)) 9267 return 0; 9268 9269 return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr, 9270 mlxsw_sp_fid_index(rif->fid), false); 9271 } 9272 9273 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif) 9274 { 9275 struct netdev_nested_priv priv = { 9276 .data = (void *)rif, 9277 }; 9278 9279 if (!netif_is_macvlan_port(rif->dev)) 9280 return 0; 9281 9282 netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n"); 9283 return netdev_walk_all_upper_dev_rcu(rif->dev, 9284 __mlxsw_sp_rif_macvlan_flush, &priv); 9285 } 9286 9287 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif, 9288 const struct mlxsw_sp_rif_params *params) 9289 { 9290 struct mlxsw_sp_rif_subport *rif_subport; 9291 9292 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9293 refcount_set(&rif_subport->ref_count, 1); 9294 rif_subport->vid = params->vid; 9295 rif_subport->lag = params->lag; 9296 if (params->lag) 9297 rif_subport->lag_id = params->lag_id; 9298 else 9299 rif_subport->system_port = params->system_port; 9300 } 9301 9302 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable) 9303 { 9304 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9305 struct mlxsw_sp_rif_subport *rif_subport; 9306 char ritr_pl[MLXSW_REG_RITR_LEN]; 9307 9308 rif_subport = mlxsw_sp_rif_subport_rif(rif); 9309 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF, 9310 rif->rif_index, rif->vr_id, rif->dev->mtu); 9311 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9312 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9313 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag, 9314 rif_subport->lag ? rif_subport->lag_id : 9315 rif_subport->system_port, 9316 rif_subport->vid); 9317 9318 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9319 } 9320 9321 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif, 9322 struct netlink_ext_ack *extack) 9323 { 9324 u8 mac_profile; 9325 int err; 9326 9327 err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr, 9328 &mac_profile, extack); 9329 if (err) 9330 return err; 9331 rif->mac_profile_id = mac_profile; 9332 9333 err = mlxsw_sp_rif_subport_op(rif, true); 9334 if (err) 9335 goto err_rif_subport_op; 9336 9337 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9338 mlxsw_sp_fid_index(rif->fid), true); 9339 if (err) 9340 goto err_rif_fdb_op; 9341 9342 mlxsw_sp_fid_rif_set(rif->fid, rif); 9343 return 0; 9344 9345 err_rif_fdb_op: 9346 mlxsw_sp_rif_subport_op(rif, false); 9347 err_rif_subport_op: 9348 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile); 9349 return err; 9350 } 9351 9352 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif) 9353 { 9354 struct mlxsw_sp_fid *fid = rif->fid; 9355 9356 mlxsw_sp_fid_rif_set(fid, NULL); 9357 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9358 mlxsw_sp_fid_index(fid), false); 9359 mlxsw_sp_rif_macvlan_flush(rif); 9360 mlxsw_sp_rif_subport_op(rif, false); 9361 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9362 } 9363 9364 static struct mlxsw_sp_fid * 9365 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif, 9366 struct netlink_ext_ack *extack) 9367 { 9368 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index); 9369 } 9370 9371 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = { 9372 .type = MLXSW_SP_RIF_TYPE_SUBPORT, 9373 .rif_size = sizeof(struct mlxsw_sp_rif_subport), 9374 .setup = mlxsw_sp_rif_subport_setup, 9375 .configure = mlxsw_sp_rif_subport_configure, 9376 .deconfigure = mlxsw_sp_rif_subport_deconfigure, 9377 .fid_get = mlxsw_sp_rif_subport_fid_get, 9378 }; 9379 9380 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable) 9381 { 9382 enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF; 9383 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9384 char ritr_pl[MLXSW_REG_RITR_LEN]; 9385 9386 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id, 9387 rif->dev->mtu); 9388 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr); 9389 mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id); 9390 mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid); 9391 9392 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9393 } 9394 9395 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) 9396 { 9397 return mlxsw_core_max_ports(mlxsw_sp->core) + 1; 9398 } 9399 9400 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif, 9401 struct netlink_ext_ack *extack) 9402 { 9403 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9404 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9405 u8 mac_profile; 9406 int err; 9407 9408 err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr, 9409 &mac_profile, extack); 9410 if (err) 9411 return err; 9412 rif->mac_profile_id = mac_profile; 9413 9414 err = mlxsw_sp_rif_fid_op(rif, fid_index, true); 9415 if (err) 9416 goto err_rif_fid_op; 9417 9418 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9419 mlxsw_sp_router_port(mlxsw_sp), true); 9420 if (err) 9421 goto err_fid_mc_flood_set; 9422 9423 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9424 mlxsw_sp_router_port(mlxsw_sp), true); 9425 if (err) 9426 goto err_fid_bc_flood_set; 9427 9428 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9429 mlxsw_sp_fid_index(rif->fid), true); 9430 if (err) 9431 goto err_rif_fdb_op; 9432 9433 mlxsw_sp_fid_rif_set(rif->fid, rif); 9434 return 0; 9435 9436 err_rif_fdb_op: 9437 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9438 mlxsw_sp_router_port(mlxsw_sp), false); 9439 err_fid_bc_flood_set: 9440 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9441 mlxsw_sp_router_port(mlxsw_sp), false); 9442 err_fid_mc_flood_set: 9443 mlxsw_sp_rif_fid_op(rif, fid_index, false); 9444 err_rif_fid_op: 9445 mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile); 9446 return err; 9447 } 9448 9449 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif) 9450 { 9451 u16 fid_index = mlxsw_sp_fid_index(rif->fid); 9452 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9453 struct mlxsw_sp_fid *fid = rif->fid; 9454 9455 mlxsw_sp_fid_rif_set(fid, NULL); 9456 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr, 9457 mlxsw_sp_fid_index(fid), false); 9458 mlxsw_sp_rif_macvlan_flush(rif); 9459 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC, 9460 mlxsw_sp_router_port(mlxsw_sp), false); 9461 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC, 9462 mlxsw_sp_router_port(mlxsw_sp), false); 9463 mlxsw_sp_rif_fid_op(rif, fid_index, false); 9464 mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id); 9465 } 9466 9467 static struct mlxsw_sp_fid * 9468 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif, 9469 struct netlink_ext_ack *extack) 9470 { 9471 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex); 9472 } 9473 9474 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9475 { 9476 struct switchdev_notifier_fdb_info info = {}; 9477 struct net_device *dev; 9478 9479 dev = br_fdb_find_port(rif->dev, mac, 0); 9480 if (!dev) 9481 return; 9482 9483 info.addr = mac; 9484 info.vid = 0; 9485 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9486 NULL); 9487 } 9488 9489 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = { 9490 .type = MLXSW_SP_RIF_TYPE_FID, 9491 .rif_size = sizeof(struct mlxsw_sp_rif), 9492 .configure = mlxsw_sp_rif_fid_configure, 9493 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9494 .fid_get = mlxsw_sp_rif_fid_fid_get, 9495 .fdb_del = mlxsw_sp_rif_fid_fdb_del, 9496 }; 9497 9498 static struct mlxsw_sp_fid * 9499 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif, 9500 struct netlink_ext_ack *extack) 9501 { 9502 struct net_device *br_dev; 9503 u16 vid; 9504 int err; 9505 9506 if (is_vlan_dev(rif->dev)) { 9507 vid = vlan_dev_vlan_id(rif->dev); 9508 br_dev = vlan_dev_real_dev(rif->dev); 9509 if (WARN_ON(!netif_is_bridge_master(br_dev))) 9510 return ERR_PTR(-EINVAL); 9511 } else { 9512 err = br_vlan_get_pvid(rif->dev, &vid); 9513 if (err < 0 || !vid) { 9514 NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID"); 9515 return ERR_PTR(-EINVAL); 9516 } 9517 } 9518 9519 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid); 9520 } 9521 9522 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac) 9523 { 9524 struct switchdev_notifier_fdb_info info = {}; 9525 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid); 9526 struct net_device *br_dev; 9527 struct net_device *dev; 9528 9529 br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev; 9530 dev = br_fdb_find_port(br_dev, mac, vid); 9531 if (!dev) 9532 return; 9533 9534 info.addr = mac; 9535 info.vid = vid; 9536 call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info, 9537 NULL); 9538 } 9539 9540 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = { 9541 .type = MLXSW_SP_RIF_TYPE_VLAN_EMU, 9542 .rif_size = sizeof(struct mlxsw_sp_rif), 9543 .configure = mlxsw_sp_rif_fid_configure, 9544 .deconfigure = mlxsw_sp_rif_fid_deconfigure, 9545 .fid_get = mlxsw_sp_rif_vlan_fid_get, 9546 .fdb_del = mlxsw_sp_rif_vlan_fdb_del, 9547 }; 9548 9549 static struct mlxsw_sp_rif_ipip_lb * 9550 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif) 9551 { 9552 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common); 9553 } 9554 9555 static void 9556 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif, 9557 const struct mlxsw_sp_rif_params *params) 9558 { 9559 struct mlxsw_sp_rif_params_ipip_lb *params_lb; 9560 struct mlxsw_sp_rif_ipip_lb *rif_lb; 9561 9562 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb, 9563 common); 9564 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif); 9565 rif_lb->lb_config = params_lb->lb_config; 9566 } 9567 9568 static int 9569 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 9570 struct netlink_ext_ack *extack) 9571 { 9572 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9573 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9574 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9575 struct mlxsw_sp_vr *ul_vr; 9576 int err; 9577 9578 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL); 9579 if (IS_ERR(ul_vr)) 9580 return PTR_ERR(ul_vr); 9581 9582 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true); 9583 if (err) 9584 goto err_loopback_op; 9585 9586 lb_rif->ul_vr_id = ul_vr->id; 9587 lb_rif->ul_rif_id = 0; 9588 ++ul_vr->rif_count; 9589 return 0; 9590 9591 err_loopback_op: 9592 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9593 return err; 9594 } 9595 9596 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9597 { 9598 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9599 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9600 struct mlxsw_sp_vr *ul_vr; 9601 9602 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id]; 9603 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false); 9604 9605 --ul_vr->rif_count; 9606 mlxsw_sp_vr_put(mlxsw_sp, ul_vr); 9607 } 9608 9609 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = { 9610 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9611 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9612 .setup = mlxsw_sp_rif_ipip_lb_setup, 9613 .configure = mlxsw_sp1_rif_ipip_lb_configure, 9614 .deconfigure = mlxsw_sp1_rif_ipip_lb_deconfigure, 9615 }; 9616 9617 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = { 9618 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9619 [MLXSW_SP_RIF_TYPE_VLAN_EMU] = &mlxsw_sp_rif_vlan_emu_ops, 9620 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9621 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp1_rif_ipip_lb_ops, 9622 }; 9623 9624 static int 9625 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable) 9626 { 9627 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9628 char ritr_pl[MLXSW_REG_RITR_LEN]; 9629 9630 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF, 9631 ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU); 9632 mlxsw_reg_ritr_loopback_protocol_set(ritr_pl, 9633 MLXSW_REG_RITR_LOOPBACK_GENERIC); 9634 9635 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); 9636 } 9637 9638 static struct mlxsw_sp_rif * 9639 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr, 9640 struct netlink_ext_ack *extack) 9641 { 9642 struct mlxsw_sp_rif *ul_rif; 9643 u16 rif_index; 9644 int err; 9645 9646 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index); 9647 if (err) { 9648 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces"); 9649 return ERR_PTR(err); 9650 } 9651 9652 ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL); 9653 if (!ul_rif) 9654 return ERR_PTR(-ENOMEM); 9655 9656 mlxsw_sp->router->rifs[rif_index] = ul_rif; 9657 ul_rif->mlxsw_sp = mlxsw_sp; 9658 err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true); 9659 if (err) 9660 goto ul_rif_op_err; 9661 9662 atomic_inc(&mlxsw_sp->router->rifs_count); 9663 return ul_rif; 9664 9665 ul_rif_op_err: 9666 mlxsw_sp->router->rifs[rif_index] = NULL; 9667 kfree(ul_rif); 9668 return ERR_PTR(err); 9669 } 9670 9671 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif) 9672 { 9673 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9674 9675 atomic_dec(&mlxsw_sp->router->rifs_count); 9676 mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false); 9677 mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL; 9678 kfree(ul_rif); 9679 } 9680 9681 static struct mlxsw_sp_rif * 9682 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, 9683 struct netlink_ext_ack *extack) 9684 { 9685 struct mlxsw_sp_vr *vr; 9686 int err; 9687 9688 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack); 9689 if (IS_ERR(vr)) 9690 return ERR_CAST(vr); 9691 9692 if (refcount_inc_not_zero(&vr->ul_rif_refcnt)) 9693 return vr->ul_rif; 9694 9695 vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack); 9696 if (IS_ERR(vr->ul_rif)) { 9697 err = PTR_ERR(vr->ul_rif); 9698 goto err_ul_rif_create; 9699 } 9700 9701 vr->rif_count++; 9702 refcount_set(&vr->ul_rif_refcnt, 1); 9703 9704 return vr->ul_rif; 9705 9706 err_ul_rif_create: 9707 mlxsw_sp_vr_put(mlxsw_sp, vr); 9708 return ERR_PTR(err); 9709 } 9710 9711 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif) 9712 { 9713 struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp; 9714 struct mlxsw_sp_vr *vr; 9715 9716 vr = &mlxsw_sp->router->vrs[ul_rif->vr_id]; 9717 9718 if (!refcount_dec_and_test(&vr->ul_rif_refcnt)) 9719 return; 9720 9721 vr->rif_count--; 9722 mlxsw_sp_ul_rif_destroy(ul_rif); 9723 mlxsw_sp_vr_put(mlxsw_sp, vr); 9724 } 9725 9726 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, 9727 u16 *ul_rif_index) 9728 { 9729 struct mlxsw_sp_rif *ul_rif; 9730 int err = 0; 9731 9732 mutex_lock(&mlxsw_sp->router->lock); 9733 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9734 if (IS_ERR(ul_rif)) { 9735 err = PTR_ERR(ul_rif); 9736 goto out; 9737 } 9738 *ul_rif_index = ul_rif->rif_index; 9739 out: 9740 mutex_unlock(&mlxsw_sp->router->lock); 9741 return err; 9742 } 9743 9744 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index) 9745 { 9746 struct mlxsw_sp_rif *ul_rif; 9747 9748 mutex_lock(&mlxsw_sp->router->lock); 9749 ul_rif = mlxsw_sp->router->rifs[ul_rif_index]; 9750 if (WARN_ON(!ul_rif)) 9751 goto out; 9752 9753 mlxsw_sp_ul_rif_put(ul_rif); 9754 out: 9755 mutex_unlock(&mlxsw_sp->router->lock); 9756 } 9757 9758 static int 9759 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif, 9760 struct netlink_ext_ack *extack) 9761 { 9762 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9763 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev); 9764 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9765 struct mlxsw_sp_rif *ul_rif; 9766 int err; 9767 9768 ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL); 9769 if (IS_ERR(ul_rif)) 9770 return PTR_ERR(ul_rif); 9771 9772 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true); 9773 if (err) 9774 goto err_loopback_op; 9775 9776 lb_rif->ul_vr_id = 0; 9777 lb_rif->ul_rif_id = ul_rif->rif_index; 9778 9779 return 0; 9780 9781 err_loopback_op: 9782 mlxsw_sp_ul_rif_put(ul_rif); 9783 return err; 9784 } 9785 9786 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif) 9787 { 9788 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif); 9789 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp; 9790 struct mlxsw_sp_rif *ul_rif; 9791 9792 ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id); 9793 mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false); 9794 mlxsw_sp_ul_rif_put(ul_rif); 9795 } 9796 9797 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = { 9798 .type = MLXSW_SP_RIF_TYPE_IPIP_LB, 9799 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb), 9800 .setup = mlxsw_sp_rif_ipip_lb_setup, 9801 .configure = mlxsw_sp2_rif_ipip_lb_configure, 9802 .deconfigure = mlxsw_sp2_rif_ipip_lb_deconfigure, 9803 }; 9804 9805 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = { 9806 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops, 9807 [MLXSW_SP_RIF_TYPE_VLAN_EMU] = &mlxsw_sp_rif_vlan_emu_ops, 9808 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops, 9809 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp2_rif_ipip_lb_ops, 9810 }; 9811 9812 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp) 9813 { 9814 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 9815 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 9816 struct mlxsw_core *core = mlxsw_sp->core; 9817 9818 if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES)) 9819 return -EIO; 9820 mlxsw_sp->router->max_rif_mac_profile = 9821 MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES); 9822 9823 mlxsw_sp->router->rifs = kcalloc(max_rifs, 9824 sizeof(struct mlxsw_sp_rif *), 9825 GFP_KERNEL); 9826 if (!mlxsw_sp->router->rifs) 9827 return -ENOMEM; 9828 9829 idr_init(&mlxsw_sp->router->rif_mac_profiles_idr); 9830 atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0); 9831 atomic_set(&mlxsw_sp->router->rifs_count, 0); 9832 devlink_resource_occ_get_register(devlink, 9833 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES, 9834 mlxsw_sp_rif_mac_profiles_occ_get, 9835 mlxsw_sp); 9836 devlink_resource_occ_get_register(devlink, 9837 MLXSW_SP_RESOURCE_RIFS, 9838 mlxsw_sp_rifs_occ_get, 9839 mlxsw_sp); 9840 9841 return 0; 9842 } 9843 9844 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp) 9845 { 9846 struct devlink *devlink = priv_to_devlink(mlxsw_sp->core); 9847 int i; 9848 9849 WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count)); 9850 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) 9851 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]); 9852 9853 devlink_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS); 9854 devlink_resource_occ_get_unregister(devlink, 9855 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES); 9856 WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr)); 9857 idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr); 9858 kfree(mlxsw_sp->router->rifs); 9859 } 9860 9861 static int 9862 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp) 9863 { 9864 char tigcr_pl[MLXSW_REG_TIGCR_LEN]; 9865 9866 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0); 9867 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl); 9868 } 9869 9870 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp) 9871 { 9872 int err; 9873 9874 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list); 9875 9876 err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp); 9877 if (err) 9878 return err; 9879 err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp); 9880 if (err) 9881 return err; 9882 9883 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp); 9884 } 9885 9886 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp) 9887 { 9888 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr; 9889 return mlxsw_sp_ipips_init(mlxsw_sp); 9890 } 9891 9892 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp) 9893 { 9894 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr; 9895 return mlxsw_sp_ipips_init(mlxsw_sp); 9896 } 9897 9898 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp) 9899 { 9900 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list)); 9901 } 9902 9903 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb) 9904 { 9905 struct mlxsw_sp_router *router; 9906 9907 /* Flush pending FIB notifications and then flush the device's 9908 * table before requesting another dump. The FIB notification 9909 * block is unregistered, so no need to take RTNL. 9910 */ 9911 mlxsw_core_flush_owq(); 9912 router = container_of(nb, struct mlxsw_sp_router, fib_nb); 9913 mlxsw_sp_router_fib_flush(router->mlxsw_sp); 9914 } 9915 9916 #ifdef CONFIG_IP_ROUTE_MULTIPATH 9917 struct mlxsw_sp_mp_hash_config { 9918 DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT); 9919 DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT); 9920 DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT); 9921 DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT); 9922 bool inc_parsing_depth; 9923 }; 9924 9925 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \ 9926 bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1) 9927 9928 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \ 9929 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1) 9930 9931 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \ 9932 bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr) 9933 9934 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config) 9935 { 9936 unsigned long *inner_headers = config->inner_headers; 9937 unsigned long *inner_fields = config->inner_fields; 9938 9939 /* IPv4 inner */ 9940 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9941 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9942 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9943 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9944 /* IPv6 inner */ 9945 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9946 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9947 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9948 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9949 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9950 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9951 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9952 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9953 } 9954 9955 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 9956 { 9957 unsigned long *headers = config->headers; 9958 unsigned long *fields = config->fields; 9959 9960 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 9961 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 9962 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 9963 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 9964 } 9965 9966 static void 9967 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config, 9968 u32 hash_fields) 9969 { 9970 unsigned long *inner_headers = config->inner_headers; 9971 unsigned long *inner_fields = config->inner_fields; 9972 9973 /* IPv4 Inner */ 9974 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP); 9975 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP); 9976 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) 9977 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4); 9978 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) 9979 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4); 9980 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9981 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL); 9982 /* IPv6 inner */ 9983 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP); 9984 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP); 9985 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) { 9986 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7); 9987 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8); 9988 } 9989 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) { 9990 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7); 9991 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8); 9992 } 9993 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO) 9994 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER); 9995 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL) 9996 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL); 9997 /* L4 inner */ 9998 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4); 9999 MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6); 10000 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT) 10001 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT); 10002 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT) 10003 MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT); 10004 } 10005 10006 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp, 10007 struct mlxsw_sp_mp_hash_config *config) 10008 { 10009 struct net *net = mlxsw_sp_net(mlxsw_sp); 10010 unsigned long *headers = config->headers; 10011 unsigned long *fields = config->fields; 10012 u32 hash_fields; 10013 10014 switch (net->ipv4.sysctl_fib_multipath_hash_policy) { 10015 case 0: 10016 mlxsw_sp_mp4_hash_outer_addr(config); 10017 break; 10018 case 1: 10019 mlxsw_sp_mp4_hash_outer_addr(config); 10020 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10021 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10022 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10023 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10024 break; 10025 case 2: 10026 /* Outer */ 10027 mlxsw_sp_mp4_hash_outer_addr(config); 10028 /* Inner */ 10029 mlxsw_sp_mp_hash_inner_l3(config); 10030 break; 10031 case 3: 10032 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields; 10033 /* Outer */ 10034 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP); 10035 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP); 10036 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4); 10037 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) 10038 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4); 10039 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) 10040 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4); 10041 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10042 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL); 10043 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10044 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10045 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10046 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10047 /* Inner */ 10048 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10049 break; 10050 } 10051 } 10052 10053 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config) 10054 { 10055 unsigned long *headers = config->headers; 10056 unsigned long *fields = config->fields; 10057 10058 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10059 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10060 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10061 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10062 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10063 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10064 } 10065 10066 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp, 10067 struct mlxsw_sp_mp_hash_config *config) 10068 { 10069 u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp)); 10070 unsigned long *headers = config->headers; 10071 unsigned long *fields = config->fields; 10072 10073 switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) { 10074 case 0: 10075 mlxsw_sp_mp6_hash_outer_addr(config); 10076 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10077 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10078 break; 10079 case 1: 10080 mlxsw_sp_mp6_hash_outer_addr(config); 10081 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10082 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10083 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10084 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10085 break; 10086 case 2: 10087 /* Outer */ 10088 mlxsw_sp_mp6_hash_outer_addr(config); 10089 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10090 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10091 /* Inner */ 10092 mlxsw_sp_mp_hash_inner_l3(config); 10093 config->inc_parsing_depth = true; 10094 break; 10095 case 3: 10096 /* Outer */ 10097 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP); 10098 MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP); 10099 MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6); 10100 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) { 10101 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7); 10102 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8); 10103 } 10104 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) { 10105 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7); 10106 MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8); 10107 } 10108 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO) 10109 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER); 10110 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL) 10111 MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL); 10112 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT) 10113 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT); 10114 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT) 10115 MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT); 10116 /* Inner */ 10117 mlxsw_sp_mp_hash_inner_custom(config, hash_fields); 10118 if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK) 10119 config->inc_parsing_depth = true; 10120 break; 10121 } 10122 } 10123 10124 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp, 10125 bool old_inc_parsing_depth, 10126 bool new_inc_parsing_depth) 10127 { 10128 int err; 10129 10130 if (!old_inc_parsing_depth && new_inc_parsing_depth) { 10131 err = mlxsw_sp_parsing_depth_inc(mlxsw_sp); 10132 if (err) 10133 return err; 10134 mlxsw_sp->router->inc_parsing_depth = true; 10135 } else if (old_inc_parsing_depth && !new_inc_parsing_depth) { 10136 mlxsw_sp_parsing_depth_dec(mlxsw_sp); 10137 mlxsw_sp->router->inc_parsing_depth = false; 10138 } 10139 10140 return 0; 10141 } 10142 10143 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10144 { 10145 bool old_inc_parsing_depth, new_inc_parsing_depth; 10146 struct mlxsw_sp_mp_hash_config config = {}; 10147 char recr2_pl[MLXSW_REG_RECR2_LEN]; 10148 unsigned long bit; 10149 u32 seed; 10150 int err; 10151 10152 seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0); 10153 mlxsw_reg_recr2_pack(recr2_pl, seed); 10154 mlxsw_sp_mp4_hash_init(mlxsw_sp, &config); 10155 mlxsw_sp_mp6_hash_init(mlxsw_sp, &config); 10156 10157 old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth; 10158 new_inc_parsing_depth = config.inc_parsing_depth; 10159 err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, 10160 old_inc_parsing_depth, 10161 new_inc_parsing_depth); 10162 if (err) 10163 return err; 10164 10165 for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT) 10166 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1); 10167 for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT) 10168 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1); 10169 for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT) 10170 mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1); 10171 for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT) 10172 mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1); 10173 10174 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl); 10175 if (err) 10176 goto err_reg_write; 10177 10178 return 0; 10179 10180 err_reg_write: 10181 mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth, 10182 old_inc_parsing_depth); 10183 return err; 10184 } 10185 #else 10186 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp) 10187 { 10188 return 0; 10189 } 10190 #endif 10191 10192 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp) 10193 { 10194 char rdpm_pl[MLXSW_REG_RDPM_LEN]; 10195 unsigned int i; 10196 10197 MLXSW_REG_ZERO(rdpm, rdpm_pl); 10198 10199 /* HW is determining switch priority based on DSCP-bits, but the 10200 * kernel is still doing that based on the ToS. Since there's a 10201 * mismatch in bits we need to make sure to translate the right 10202 * value ToS would observe, skipping the 2 least-significant ECN bits. 10203 */ 10204 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++) 10205 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2)); 10206 10207 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl); 10208 } 10209 10210 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp) 10211 { 10212 struct net *net = mlxsw_sp_net(mlxsw_sp); 10213 bool usp = net->ipv4.sysctl_ip_fwd_update_priority; 10214 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10215 u64 max_rifs; 10216 10217 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS)) 10218 return -EIO; 10219 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); 10220 10221 mlxsw_reg_rgcr_pack(rgcr_pl, true, true); 10222 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs); 10223 mlxsw_reg_rgcr_usp_set(rgcr_pl, usp); 10224 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10225 } 10226 10227 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10228 { 10229 char rgcr_pl[MLXSW_REG_RGCR_LEN]; 10230 10231 mlxsw_reg_rgcr_pack(rgcr_pl, false, false); 10232 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl); 10233 } 10234 10235 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp) 10236 { 10237 u16 lb_rif_index; 10238 int err; 10239 10240 /* Create a generic loopback RIF associated with the main table 10241 * (default VRF). Any table can be used, but the main table exists 10242 * anyway, so we do not waste resources. 10243 */ 10244 err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, 10245 &lb_rif_index); 10246 if (err) 10247 return err; 10248 10249 mlxsw_sp->router->lb_rif_index = lb_rif_index; 10250 10251 return 0; 10252 } 10253 10254 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp) 10255 { 10256 mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index); 10257 } 10258 10259 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp) 10260 { 10261 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges); 10262 10263 mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr; 10264 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges; 10265 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10266 10267 return 0; 10268 } 10269 10270 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = { 10271 .init = mlxsw_sp1_router_init, 10272 .ipips_init = mlxsw_sp1_ipips_init, 10273 }; 10274 10275 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp) 10276 { 10277 size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges); 10278 10279 mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr; 10280 mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges; 10281 mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count; 10282 10283 return 0; 10284 } 10285 10286 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = { 10287 .init = mlxsw_sp2_router_init, 10288 .ipips_init = mlxsw_sp2_ipips_init, 10289 }; 10290 10291 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp, 10292 struct netlink_ext_ack *extack) 10293 { 10294 struct mlxsw_sp_router *router; 10295 int err; 10296 10297 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL); 10298 if (!router) 10299 return -ENOMEM; 10300 mutex_init(&router->lock); 10301 mlxsw_sp->router = router; 10302 router->mlxsw_sp = mlxsw_sp; 10303 10304 err = mlxsw_sp->router_ops->init(mlxsw_sp); 10305 if (err) 10306 goto err_router_ops_init; 10307 10308 INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list); 10309 INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw, 10310 mlxsw_sp_nh_grp_activity_work); 10311 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list); 10312 err = __mlxsw_sp_router_init(mlxsw_sp); 10313 if (err) 10314 goto err_router_init; 10315 10316 err = mlxsw_sp_rifs_init(mlxsw_sp); 10317 if (err) 10318 goto err_rifs_init; 10319 10320 err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp); 10321 if (err) 10322 goto err_ipips_init; 10323 10324 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht, 10325 &mlxsw_sp_nexthop_ht_params); 10326 if (err) 10327 goto err_nexthop_ht_init; 10328 10329 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht, 10330 &mlxsw_sp_nexthop_group_ht_params); 10331 if (err) 10332 goto err_nexthop_group_ht_init; 10333 10334 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list); 10335 err = mlxsw_sp_lpm_init(mlxsw_sp); 10336 if (err) 10337 goto err_lpm_init; 10338 10339 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops); 10340 if (err) 10341 goto err_mr_init; 10342 10343 err = mlxsw_sp_vrs_init(mlxsw_sp); 10344 if (err) 10345 goto err_vrs_init; 10346 10347 err = mlxsw_sp_lb_rif_init(mlxsw_sp); 10348 if (err) 10349 goto err_lb_rif_init; 10350 10351 err = mlxsw_sp_neigh_init(mlxsw_sp); 10352 if (err) 10353 goto err_neigh_init; 10354 10355 err = mlxsw_sp_mp_hash_init(mlxsw_sp); 10356 if (err) 10357 goto err_mp_hash_init; 10358 10359 err = mlxsw_sp_dscp_init(mlxsw_sp); 10360 if (err) 10361 goto err_dscp_init; 10362 10363 router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event; 10364 err = register_inetaddr_notifier(&router->inetaddr_nb); 10365 if (err) 10366 goto err_register_inetaddr_notifier; 10367 10368 router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event; 10369 err = register_inet6addr_notifier(&router->inet6addr_nb); 10370 if (err) 10371 goto err_register_inet6addr_notifier; 10372 10373 mlxsw_sp->router->netevent_nb.notifier_call = 10374 mlxsw_sp_router_netevent_event; 10375 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10376 if (err) 10377 goto err_register_netevent_notifier; 10378 10379 mlxsw_sp->router->nexthop_nb.notifier_call = 10380 mlxsw_sp_nexthop_obj_event; 10381 err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10382 &mlxsw_sp->router->nexthop_nb, 10383 extack); 10384 if (err) 10385 goto err_register_nexthop_notifier; 10386 10387 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event; 10388 err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10389 &mlxsw_sp->router->fib_nb, 10390 mlxsw_sp_router_fib_dump_flush, extack); 10391 if (err) 10392 goto err_register_fib_notifier; 10393 10394 mlxsw_sp->router->netdevice_nb.notifier_call = 10395 mlxsw_sp_router_netdevice_event; 10396 err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10397 &mlxsw_sp->router->netdevice_nb); 10398 if (err) 10399 goto err_register_netdev_notifier; 10400 10401 return 0; 10402 10403 err_register_netdev_notifier: 10404 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10405 &mlxsw_sp->router->fib_nb); 10406 err_register_fib_notifier: 10407 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10408 &mlxsw_sp->router->nexthop_nb); 10409 err_register_nexthop_notifier: 10410 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10411 err_register_netevent_notifier: 10412 unregister_inet6addr_notifier(&router->inet6addr_nb); 10413 err_register_inet6addr_notifier: 10414 unregister_inetaddr_notifier(&router->inetaddr_nb); 10415 err_register_inetaddr_notifier: 10416 mlxsw_core_flush_owq(); 10417 err_dscp_init: 10418 err_mp_hash_init: 10419 mlxsw_sp_neigh_fini(mlxsw_sp); 10420 err_neigh_init: 10421 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10422 err_lb_rif_init: 10423 mlxsw_sp_vrs_fini(mlxsw_sp); 10424 err_vrs_init: 10425 mlxsw_sp_mr_fini(mlxsw_sp); 10426 err_mr_init: 10427 mlxsw_sp_lpm_fini(mlxsw_sp); 10428 err_lpm_init: 10429 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10430 err_nexthop_group_ht_init: 10431 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10432 err_nexthop_ht_init: 10433 mlxsw_sp_ipips_fini(mlxsw_sp); 10434 err_ipips_init: 10435 mlxsw_sp_rifs_fini(mlxsw_sp); 10436 err_rifs_init: 10437 __mlxsw_sp_router_fini(mlxsw_sp); 10438 err_router_init: 10439 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10440 err_router_ops_init: 10441 mutex_destroy(&mlxsw_sp->router->lock); 10442 kfree(mlxsw_sp->router); 10443 return err; 10444 } 10445 10446 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp) 10447 { 10448 unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp), 10449 &mlxsw_sp->router->netdevice_nb); 10450 unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), 10451 &mlxsw_sp->router->fib_nb); 10452 unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp), 10453 &mlxsw_sp->router->nexthop_nb); 10454 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb); 10455 unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb); 10456 unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb); 10457 mlxsw_core_flush_owq(); 10458 mlxsw_sp_neigh_fini(mlxsw_sp); 10459 mlxsw_sp_lb_rif_fini(mlxsw_sp); 10460 mlxsw_sp_vrs_fini(mlxsw_sp); 10461 mlxsw_sp_mr_fini(mlxsw_sp); 10462 mlxsw_sp_lpm_fini(mlxsw_sp); 10463 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht); 10464 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht); 10465 mlxsw_sp_ipips_fini(mlxsw_sp); 10466 mlxsw_sp_rifs_fini(mlxsw_sp); 10467 __mlxsw_sp_router_fini(mlxsw_sp); 10468 cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw); 10469 mutex_destroy(&mlxsw_sp->router->lock); 10470 kfree(mlxsw_sp->router); 10471 } 10472