1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/rhashtable.h>
7 #include <linux/bitops.h>
8 #include <linux/in6.h>
9 #include <linux/notifier.h>
10 #include <linux/inetdevice.h>
11 #include <linux/netdevice.h>
12 #include <linux/if_bridge.h>
13 #include <linux/socket.h>
14 #include <linux/route.h>
15 #include <linux/gcd.h>
16 #include <linux/if_macvlan.h>
17 #include <linux/refcount.h>
18 #include <linux/jhash.h>
19 #include <linux/net_namespace.h>
20 #include <linux/mutex.h>
21 #include <linux/genalloc.h>
22 #include <net/netevent.h>
23 #include <net/neighbour.h>
24 #include <net/arp.h>
25 #include <net/inet_dscp.h>
26 #include <net/ip_fib.h>
27 #include <net/ip6_fib.h>
28 #include <net/nexthop.h>
29 #include <net/fib_rules.h>
30 #include <net/ip_tunnels.h>
31 #include <net/l3mdev.h>
32 #include <net/addrconf.h>
33 #include <net/ndisc.h>
34 #include <net/ipv6.h>
35 #include <net/fib_notifier.h>
36 #include <net/switchdev.h>
37 
38 #include "spectrum.h"
39 #include "core.h"
40 #include "reg.h"
41 #include "spectrum_cnt.h"
42 #include "spectrum_dpipe.h"
43 #include "spectrum_ipip.h"
44 #include "spectrum_mr.h"
45 #include "spectrum_mr_tcam.h"
46 #include "spectrum_router.h"
47 #include "spectrum_span.h"
48 
49 struct mlxsw_sp_fib;
50 struct mlxsw_sp_vr;
51 struct mlxsw_sp_lpm_tree;
52 struct mlxsw_sp_rif_ops;
53 
54 struct mlxsw_sp_rif {
55 	struct list_head nexthop_list;
56 	struct list_head neigh_list;
57 	struct net_device *dev; /* NULL for underlay RIF */
58 	struct mlxsw_sp_fid *fid;
59 	unsigned char addr[ETH_ALEN];
60 	int mtu;
61 	u16 rif_index;
62 	u8 mac_profile_id;
63 	u8 rif_entries;
64 	u16 vr_id;
65 	const struct mlxsw_sp_rif_ops *ops;
66 	struct mlxsw_sp *mlxsw_sp;
67 
68 	unsigned int counter_ingress;
69 	bool counter_ingress_valid;
70 	unsigned int counter_egress;
71 	bool counter_egress_valid;
72 };
73 
74 struct mlxsw_sp_rif_params {
75 	struct net_device *dev;
76 	union {
77 		u16 system_port;
78 		u16 lag_id;
79 	};
80 	u16 vid;
81 	bool lag;
82 	bool double_entry;
83 };
84 
85 struct mlxsw_sp_rif_subport {
86 	struct mlxsw_sp_rif common;
87 	refcount_t ref_count;
88 	union {
89 		u16 system_port;
90 		u16 lag_id;
91 	};
92 	u16 vid;
93 	bool lag;
94 };
95 
96 struct mlxsw_sp_rif_ipip_lb {
97 	struct mlxsw_sp_rif common;
98 	struct mlxsw_sp_rif_ipip_lb_config lb_config;
99 	u16 ul_vr_id; /* Reserved for Spectrum-2. */
100 	u16 ul_rif_id; /* Reserved for Spectrum. */
101 };
102 
103 struct mlxsw_sp_rif_params_ipip_lb {
104 	struct mlxsw_sp_rif_params common;
105 	struct mlxsw_sp_rif_ipip_lb_config lb_config;
106 };
107 
108 struct mlxsw_sp_rif_ops {
109 	enum mlxsw_sp_rif_type type;
110 	size_t rif_size;
111 
112 	void (*setup)(struct mlxsw_sp_rif *rif,
113 		      const struct mlxsw_sp_rif_params *params);
114 	int (*configure)(struct mlxsw_sp_rif *rif,
115 			 struct netlink_ext_ack *extack);
116 	void (*deconfigure)(struct mlxsw_sp_rif *rif);
117 	struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
118 					 struct netlink_ext_ack *extack);
119 	void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
120 };
121 
122 struct mlxsw_sp_rif_mac_profile {
123 	unsigned char mac_prefix[ETH_ALEN];
124 	refcount_t ref_count;
125 	u8 id;
126 };
127 
128 struct mlxsw_sp_router_ops {
129 	int (*init)(struct mlxsw_sp *mlxsw_sp);
130 	int (*ipips_init)(struct mlxsw_sp *mlxsw_sp);
131 };
132 
133 static struct mlxsw_sp_rif *
134 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
135 			 const struct net_device *dev);
136 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
137 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
138 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
139 				  struct mlxsw_sp_lpm_tree *lpm_tree);
140 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
141 				     const struct mlxsw_sp_fib *fib,
142 				     u8 tree_id);
143 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
144 				       const struct mlxsw_sp_fib *fib);
145 
146 static unsigned int *
147 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
148 			   enum mlxsw_sp_rif_counter_dir dir)
149 {
150 	switch (dir) {
151 	case MLXSW_SP_RIF_COUNTER_EGRESS:
152 		return &rif->counter_egress;
153 	case MLXSW_SP_RIF_COUNTER_INGRESS:
154 		return &rif->counter_ingress;
155 	}
156 	return NULL;
157 }
158 
159 static bool
160 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
161 			       enum mlxsw_sp_rif_counter_dir dir)
162 {
163 	switch (dir) {
164 	case MLXSW_SP_RIF_COUNTER_EGRESS:
165 		return rif->counter_egress_valid;
166 	case MLXSW_SP_RIF_COUNTER_INGRESS:
167 		return rif->counter_ingress_valid;
168 	}
169 	return false;
170 }
171 
172 static void
173 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
174 			       enum mlxsw_sp_rif_counter_dir dir,
175 			       bool valid)
176 {
177 	switch (dir) {
178 	case MLXSW_SP_RIF_COUNTER_EGRESS:
179 		rif->counter_egress_valid = valid;
180 		break;
181 	case MLXSW_SP_RIF_COUNTER_INGRESS:
182 		rif->counter_ingress_valid = valid;
183 		break;
184 	}
185 }
186 
187 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
188 				     unsigned int counter_index, bool enable,
189 				     enum mlxsw_sp_rif_counter_dir dir)
190 {
191 	char ritr_pl[MLXSW_REG_RITR_LEN];
192 	bool is_egress = false;
193 	int err;
194 
195 	if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
196 		is_egress = true;
197 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
198 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
199 	if (err)
200 		return err;
201 
202 	mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
203 				    is_egress);
204 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
205 }
206 
207 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
208 				   struct mlxsw_sp_rif *rif,
209 				   enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
210 {
211 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
212 	unsigned int *p_counter_index;
213 	bool valid;
214 	int err;
215 
216 	valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
217 	if (!valid)
218 		return -EINVAL;
219 
220 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
221 	if (!p_counter_index)
222 		return -EINVAL;
223 	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
224 			     MLXSW_REG_RICNT_OPCODE_NOP);
225 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
226 	if (err)
227 		return err;
228 	*cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
229 	return 0;
230 }
231 
232 struct mlxsw_sp_rif_counter_set_basic {
233 	u64 good_unicast_packets;
234 	u64 good_multicast_packets;
235 	u64 good_broadcast_packets;
236 	u64 good_unicast_bytes;
237 	u64 good_multicast_bytes;
238 	u64 good_broadcast_bytes;
239 	u64 error_packets;
240 	u64 discard_packets;
241 	u64 error_bytes;
242 	u64 discard_bytes;
243 };
244 
245 static int
246 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif,
247 				 enum mlxsw_sp_rif_counter_dir dir,
248 				 struct mlxsw_sp_rif_counter_set_basic *set)
249 {
250 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
251 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
252 	unsigned int *p_counter_index;
253 	int err;
254 
255 	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
256 		return -EINVAL;
257 
258 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
259 	if (!p_counter_index)
260 		return -EINVAL;
261 
262 	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
263 			     MLXSW_REG_RICNT_OPCODE_CLEAR);
264 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
265 	if (err)
266 		return err;
267 
268 	if (!set)
269 		return 0;
270 
271 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME)				\
272 		(set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl))
273 
274 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets);
275 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets);
276 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets);
277 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes);
278 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes);
279 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes);
280 	MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets);
281 	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets);
282 	MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes);
283 	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes);
284 
285 #undef MLXSW_SP_RIF_COUNTER_EXTRACT
286 
287 	return 0;
288 }
289 
290 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
291 				      unsigned int counter_index)
292 {
293 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
294 
295 	mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
296 			     MLXSW_REG_RICNT_OPCODE_CLEAR);
297 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
298 }
299 
300 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif,
301 			       enum mlxsw_sp_rif_counter_dir dir)
302 {
303 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
304 	unsigned int *p_counter_index;
305 	int err;
306 
307 	if (mlxsw_sp_rif_counter_valid_get(rif, dir))
308 		return 0;
309 
310 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
311 	if (!p_counter_index)
312 		return -EINVAL;
313 
314 	err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
315 				     p_counter_index);
316 	if (err)
317 		return err;
318 
319 	err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
320 	if (err)
321 		goto err_counter_clear;
322 
323 	err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
324 					*p_counter_index, true, dir);
325 	if (err)
326 		goto err_counter_edit;
327 	mlxsw_sp_rif_counter_valid_set(rif, dir, true);
328 	return 0;
329 
330 err_counter_edit:
331 err_counter_clear:
332 	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
333 			      *p_counter_index);
334 	return err;
335 }
336 
337 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif,
338 			       enum mlxsw_sp_rif_counter_dir dir)
339 {
340 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
341 	unsigned int *p_counter_index;
342 
343 	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
344 		return;
345 
346 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
347 	if (WARN_ON(!p_counter_index))
348 		return;
349 	mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
350 				  *p_counter_index, false, dir);
351 	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
352 			      *p_counter_index);
353 	mlxsw_sp_rif_counter_valid_set(rif, dir, false);
354 }
355 
356 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
357 {
358 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
359 	struct devlink *devlink;
360 
361 	devlink = priv_to_devlink(mlxsw_sp->core);
362 	if (!devlink_dpipe_table_counter_enabled(devlink,
363 						 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
364 		return;
365 	mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
366 }
367 
368 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
369 {
370 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
371 }
372 
373 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
374 
375 struct mlxsw_sp_prefix_usage {
376 	DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
377 };
378 
379 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
380 	for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
381 
382 static bool
383 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
384 			 struct mlxsw_sp_prefix_usage *prefix_usage2)
385 {
386 	return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
387 }
388 
389 static void
390 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
391 			  struct mlxsw_sp_prefix_usage *prefix_usage2)
392 {
393 	memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
394 }
395 
396 static void
397 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
398 			  unsigned char prefix_len)
399 {
400 	set_bit(prefix_len, prefix_usage->b);
401 }
402 
403 static void
404 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
405 			    unsigned char prefix_len)
406 {
407 	clear_bit(prefix_len, prefix_usage->b);
408 }
409 
410 struct mlxsw_sp_fib_key {
411 	unsigned char addr[sizeof(struct in6_addr)];
412 	unsigned char prefix_len;
413 };
414 
415 enum mlxsw_sp_fib_entry_type {
416 	MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
417 	MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
418 	MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
419 	MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
420 	MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
421 
422 	/* This is a special case of local delivery, where a packet should be
423 	 * decapsulated on reception. Note that there is no corresponding ENCAP,
424 	 * because that's a type of next hop, not of FIB entry. (There can be
425 	 * several next hops in a REMOTE entry, and some of them may be
426 	 * encapsulating entries.)
427 	 */
428 	MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
429 	MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
430 };
431 
432 struct mlxsw_sp_nexthop_group_info;
433 struct mlxsw_sp_nexthop_group;
434 struct mlxsw_sp_fib_entry;
435 
436 struct mlxsw_sp_fib_node {
437 	struct mlxsw_sp_fib_entry *fib_entry;
438 	struct list_head list;
439 	struct rhash_head ht_node;
440 	struct mlxsw_sp_fib *fib;
441 	struct mlxsw_sp_fib_key key;
442 };
443 
444 struct mlxsw_sp_fib_entry_decap {
445 	struct mlxsw_sp_ipip_entry *ipip_entry;
446 	u32 tunnel_index;
447 };
448 
449 struct mlxsw_sp_fib_entry {
450 	struct mlxsw_sp_fib_node *fib_node;
451 	enum mlxsw_sp_fib_entry_type type;
452 	struct list_head nexthop_group_node;
453 	struct mlxsw_sp_nexthop_group *nh_group;
454 	struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
455 };
456 
457 struct mlxsw_sp_fib4_entry {
458 	struct mlxsw_sp_fib_entry common;
459 	struct fib_info *fi;
460 	u32 tb_id;
461 	dscp_t dscp;
462 	u8 type;
463 };
464 
465 struct mlxsw_sp_fib6_entry {
466 	struct mlxsw_sp_fib_entry common;
467 	struct list_head rt6_list;
468 	unsigned int nrt6;
469 };
470 
471 struct mlxsw_sp_rt6 {
472 	struct list_head list;
473 	struct fib6_info *rt;
474 };
475 
476 struct mlxsw_sp_lpm_tree {
477 	u8 id; /* tree ID */
478 	unsigned int ref_count;
479 	enum mlxsw_sp_l3proto proto;
480 	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
481 	struct mlxsw_sp_prefix_usage prefix_usage;
482 };
483 
484 struct mlxsw_sp_fib {
485 	struct rhashtable ht;
486 	struct list_head node_list;
487 	struct mlxsw_sp_vr *vr;
488 	struct mlxsw_sp_lpm_tree *lpm_tree;
489 	enum mlxsw_sp_l3proto proto;
490 };
491 
492 struct mlxsw_sp_vr {
493 	u16 id; /* virtual router ID */
494 	u32 tb_id; /* kernel fib table id */
495 	unsigned int rif_count;
496 	struct mlxsw_sp_fib *fib4;
497 	struct mlxsw_sp_fib *fib6;
498 	struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
499 	struct mlxsw_sp_rif *ul_rif;
500 	refcount_t ul_rif_refcnt;
501 };
502 
503 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
504 
505 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
506 						struct mlxsw_sp_vr *vr,
507 						enum mlxsw_sp_l3proto proto)
508 {
509 	struct mlxsw_sp_lpm_tree *lpm_tree;
510 	struct mlxsw_sp_fib *fib;
511 	int err;
512 
513 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
514 	fib = kzalloc(sizeof(*fib), GFP_KERNEL);
515 	if (!fib)
516 		return ERR_PTR(-ENOMEM);
517 	err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
518 	if (err)
519 		goto err_rhashtable_init;
520 	INIT_LIST_HEAD(&fib->node_list);
521 	fib->proto = proto;
522 	fib->vr = vr;
523 	fib->lpm_tree = lpm_tree;
524 	mlxsw_sp_lpm_tree_hold(lpm_tree);
525 	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
526 	if (err)
527 		goto err_lpm_tree_bind;
528 	return fib;
529 
530 err_lpm_tree_bind:
531 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
532 err_rhashtable_init:
533 	kfree(fib);
534 	return ERR_PTR(err);
535 }
536 
537 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
538 				 struct mlxsw_sp_fib *fib)
539 {
540 	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
541 	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
542 	WARN_ON(!list_empty(&fib->node_list));
543 	rhashtable_destroy(&fib->ht);
544 	kfree(fib);
545 }
546 
547 static struct mlxsw_sp_lpm_tree *
548 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
549 {
550 	static struct mlxsw_sp_lpm_tree *lpm_tree;
551 	int i;
552 
553 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
554 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
555 		if (lpm_tree->ref_count == 0)
556 			return lpm_tree;
557 	}
558 	return NULL;
559 }
560 
561 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
562 				   struct mlxsw_sp_lpm_tree *lpm_tree)
563 {
564 	char ralta_pl[MLXSW_REG_RALTA_LEN];
565 
566 	mlxsw_reg_ralta_pack(ralta_pl, true,
567 			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
568 			     lpm_tree->id);
569 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
570 }
571 
572 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
573 				   struct mlxsw_sp_lpm_tree *lpm_tree)
574 {
575 	char ralta_pl[MLXSW_REG_RALTA_LEN];
576 
577 	mlxsw_reg_ralta_pack(ralta_pl, false,
578 			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
579 			     lpm_tree->id);
580 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
581 }
582 
583 static int
584 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
585 				  struct mlxsw_sp_prefix_usage *prefix_usage,
586 				  struct mlxsw_sp_lpm_tree *lpm_tree)
587 {
588 	char ralst_pl[MLXSW_REG_RALST_LEN];
589 	u8 root_bin = 0;
590 	u8 prefix;
591 	u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
592 
593 	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
594 		root_bin = prefix;
595 
596 	mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
597 	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
598 		if (prefix == 0)
599 			continue;
600 		mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
601 					 MLXSW_REG_RALST_BIN_NO_CHILD);
602 		last_prefix = prefix;
603 	}
604 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
605 }
606 
607 static struct mlxsw_sp_lpm_tree *
608 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
609 			 struct mlxsw_sp_prefix_usage *prefix_usage,
610 			 enum mlxsw_sp_l3proto proto)
611 {
612 	struct mlxsw_sp_lpm_tree *lpm_tree;
613 	int err;
614 
615 	lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
616 	if (!lpm_tree)
617 		return ERR_PTR(-EBUSY);
618 	lpm_tree->proto = proto;
619 	err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
620 	if (err)
621 		return ERR_PTR(err);
622 
623 	err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
624 						lpm_tree);
625 	if (err)
626 		goto err_left_struct_set;
627 	memcpy(&lpm_tree->prefix_usage, prefix_usage,
628 	       sizeof(lpm_tree->prefix_usage));
629 	memset(&lpm_tree->prefix_ref_count, 0,
630 	       sizeof(lpm_tree->prefix_ref_count));
631 	lpm_tree->ref_count = 1;
632 	return lpm_tree;
633 
634 err_left_struct_set:
635 	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
636 	return ERR_PTR(err);
637 }
638 
639 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
640 				      struct mlxsw_sp_lpm_tree *lpm_tree)
641 {
642 	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
643 }
644 
645 static struct mlxsw_sp_lpm_tree *
646 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
647 		      struct mlxsw_sp_prefix_usage *prefix_usage,
648 		      enum mlxsw_sp_l3proto proto)
649 {
650 	struct mlxsw_sp_lpm_tree *lpm_tree;
651 	int i;
652 
653 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
654 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
655 		if (lpm_tree->ref_count != 0 &&
656 		    lpm_tree->proto == proto &&
657 		    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
658 					     prefix_usage)) {
659 			mlxsw_sp_lpm_tree_hold(lpm_tree);
660 			return lpm_tree;
661 		}
662 	}
663 	return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
664 }
665 
666 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
667 {
668 	lpm_tree->ref_count++;
669 }
670 
671 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
672 				  struct mlxsw_sp_lpm_tree *lpm_tree)
673 {
674 	if (--lpm_tree->ref_count == 0)
675 		mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
676 }
677 
678 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
679 
680 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
681 {
682 	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
683 	struct mlxsw_sp_lpm_tree *lpm_tree;
684 	u64 max_trees;
685 	int err, i;
686 
687 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
688 		return -EIO;
689 
690 	max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
691 	mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
692 	mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
693 					     sizeof(struct mlxsw_sp_lpm_tree),
694 					     GFP_KERNEL);
695 	if (!mlxsw_sp->router->lpm.trees)
696 		return -ENOMEM;
697 
698 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
699 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
700 		lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
701 	}
702 
703 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
704 					 MLXSW_SP_L3_PROTO_IPV4);
705 	if (IS_ERR(lpm_tree)) {
706 		err = PTR_ERR(lpm_tree);
707 		goto err_ipv4_tree_get;
708 	}
709 	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
710 
711 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
712 					 MLXSW_SP_L3_PROTO_IPV6);
713 	if (IS_ERR(lpm_tree)) {
714 		err = PTR_ERR(lpm_tree);
715 		goto err_ipv6_tree_get;
716 	}
717 	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
718 
719 	return 0;
720 
721 err_ipv6_tree_get:
722 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
723 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
724 err_ipv4_tree_get:
725 	kfree(mlxsw_sp->router->lpm.trees);
726 	return err;
727 }
728 
729 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
730 {
731 	struct mlxsw_sp_lpm_tree *lpm_tree;
732 
733 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
734 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
735 
736 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
737 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
738 
739 	kfree(mlxsw_sp->router->lpm.trees);
740 }
741 
742 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
743 {
744 	return !!vr->fib4 || !!vr->fib6 ||
745 	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
746 	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
747 }
748 
749 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
750 {
751 	struct mlxsw_sp_vr *vr;
752 	int i;
753 
754 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
755 		vr = &mlxsw_sp->router->vrs[i];
756 		if (!mlxsw_sp_vr_is_used(vr))
757 			return vr;
758 	}
759 	return NULL;
760 }
761 
762 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
763 				     const struct mlxsw_sp_fib *fib, u8 tree_id)
764 {
765 	char raltb_pl[MLXSW_REG_RALTB_LEN];
766 
767 	mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
768 			     (enum mlxsw_reg_ralxx_protocol) fib->proto,
769 			     tree_id);
770 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
771 }
772 
773 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
774 				       const struct mlxsw_sp_fib *fib)
775 {
776 	char raltb_pl[MLXSW_REG_RALTB_LEN];
777 
778 	/* Bind to tree 0 which is default */
779 	mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
780 			     (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
781 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
782 }
783 
784 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
785 {
786 	/* For our purpose, squash main, default and local tables into one */
787 	if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
788 		tb_id = RT_TABLE_MAIN;
789 	return tb_id;
790 }
791 
792 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
793 					    u32 tb_id)
794 {
795 	struct mlxsw_sp_vr *vr;
796 	int i;
797 
798 	tb_id = mlxsw_sp_fix_tb_id(tb_id);
799 
800 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
801 		vr = &mlxsw_sp->router->vrs[i];
802 		if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
803 			return vr;
804 	}
805 	return NULL;
806 }
807 
808 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
809 				u16 *vr_id)
810 {
811 	struct mlxsw_sp_vr *vr;
812 	int err = 0;
813 
814 	mutex_lock(&mlxsw_sp->router->lock);
815 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
816 	if (!vr) {
817 		err = -ESRCH;
818 		goto out;
819 	}
820 	*vr_id = vr->id;
821 out:
822 	mutex_unlock(&mlxsw_sp->router->lock);
823 	return err;
824 }
825 
826 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
827 					    enum mlxsw_sp_l3proto proto)
828 {
829 	switch (proto) {
830 	case MLXSW_SP_L3_PROTO_IPV4:
831 		return vr->fib4;
832 	case MLXSW_SP_L3_PROTO_IPV6:
833 		return vr->fib6;
834 	}
835 	return NULL;
836 }
837 
838 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
839 					      u32 tb_id,
840 					      struct netlink_ext_ack *extack)
841 {
842 	struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
843 	struct mlxsw_sp_fib *fib4;
844 	struct mlxsw_sp_fib *fib6;
845 	struct mlxsw_sp_vr *vr;
846 	int err;
847 
848 	vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
849 	if (!vr) {
850 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
851 		return ERR_PTR(-EBUSY);
852 	}
853 	fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
854 	if (IS_ERR(fib4))
855 		return ERR_CAST(fib4);
856 	fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
857 	if (IS_ERR(fib6)) {
858 		err = PTR_ERR(fib6);
859 		goto err_fib6_create;
860 	}
861 	mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
862 					     MLXSW_SP_L3_PROTO_IPV4);
863 	if (IS_ERR(mr4_table)) {
864 		err = PTR_ERR(mr4_table);
865 		goto err_mr4_table_create;
866 	}
867 	mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
868 					     MLXSW_SP_L3_PROTO_IPV6);
869 	if (IS_ERR(mr6_table)) {
870 		err = PTR_ERR(mr6_table);
871 		goto err_mr6_table_create;
872 	}
873 
874 	vr->fib4 = fib4;
875 	vr->fib6 = fib6;
876 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
877 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
878 	vr->tb_id = tb_id;
879 	return vr;
880 
881 err_mr6_table_create:
882 	mlxsw_sp_mr_table_destroy(mr4_table);
883 err_mr4_table_create:
884 	mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
885 err_fib6_create:
886 	mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
887 	return ERR_PTR(err);
888 }
889 
890 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
891 				struct mlxsw_sp_vr *vr)
892 {
893 	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
894 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
895 	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
896 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
897 	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
898 	vr->fib6 = NULL;
899 	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
900 	vr->fib4 = NULL;
901 }
902 
903 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
904 					   struct netlink_ext_ack *extack)
905 {
906 	struct mlxsw_sp_vr *vr;
907 
908 	tb_id = mlxsw_sp_fix_tb_id(tb_id);
909 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
910 	if (!vr)
911 		vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
912 	return vr;
913 }
914 
915 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
916 {
917 	if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
918 	    list_empty(&vr->fib6->node_list) &&
919 	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
920 	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
921 		mlxsw_sp_vr_destroy(mlxsw_sp, vr);
922 }
923 
924 static bool
925 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
926 				    enum mlxsw_sp_l3proto proto, u8 tree_id)
927 {
928 	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
929 
930 	if (!mlxsw_sp_vr_is_used(vr))
931 		return false;
932 	if (fib->lpm_tree->id == tree_id)
933 		return true;
934 	return false;
935 }
936 
937 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
938 					struct mlxsw_sp_fib *fib,
939 					struct mlxsw_sp_lpm_tree *new_tree)
940 {
941 	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
942 	int err;
943 
944 	fib->lpm_tree = new_tree;
945 	mlxsw_sp_lpm_tree_hold(new_tree);
946 	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
947 	if (err)
948 		goto err_tree_bind;
949 	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
950 	return 0;
951 
952 err_tree_bind:
953 	mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
954 	fib->lpm_tree = old_tree;
955 	return err;
956 }
957 
958 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
959 					 struct mlxsw_sp_fib *fib,
960 					 struct mlxsw_sp_lpm_tree *new_tree)
961 {
962 	enum mlxsw_sp_l3proto proto = fib->proto;
963 	struct mlxsw_sp_lpm_tree *old_tree;
964 	u8 old_id, new_id = new_tree->id;
965 	struct mlxsw_sp_vr *vr;
966 	int i, err;
967 
968 	old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
969 	old_id = old_tree->id;
970 
971 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
972 		vr = &mlxsw_sp->router->vrs[i];
973 		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
974 			continue;
975 		err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
976 						   mlxsw_sp_vr_fib(vr, proto),
977 						   new_tree);
978 		if (err)
979 			goto err_tree_replace;
980 	}
981 
982 	memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
983 	       sizeof(new_tree->prefix_ref_count));
984 	mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
985 	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
986 
987 	return 0;
988 
989 err_tree_replace:
990 	for (i--; i >= 0; i--) {
991 		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
992 			continue;
993 		mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
994 					     mlxsw_sp_vr_fib(vr, proto),
995 					     old_tree);
996 	}
997 	return err;
998 }
999 
1000 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1001 {
1002 	struct mlxsw_sp_vr *vr;
1003 	u64 max_vrs;
1004 	int i;
1005 
1006 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1007 		return -EIO;
1008 
1009 	max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1010 	mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1011 					GFP_KERNEL);
1012 	if (!mlxsw_sp->router->vrs)
1013 		return -ENOMEM;
1014 
1015 	for (i = 0; i < max_vrs; i++) {
1016 		vr = &mlxsw_sp->router->vrs[i];
1017 		vr->id = i;
1018 	}
1019 
1020 	return 0;
1021 }
1022 
1023 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1024 
1025 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1026 {
1027 	/* At this stage we're guaranteed not to have new incoming
1028 	 * FIB notifications and the work queue is free from FIBs
1029 	 * sitting on top of mlxsw netdevs. However, we can still
1030 	 * have other FIBs queued. Flush the queue before flushing
1031 	 * the device's tables. No need for locks, as we're the only
1032 	 * writer.
1033 	 */
1034 	mlxsw_core_flush_owq();
1035 	mlxsw_sp_router_fib_flush(mlxsw_sp);
1036 	kfree(mlxsw_sp->router->vrs);
1037 }
1038 
1039 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1040 {
1041 	struct net_device *d;
1042 	u32 tb_id;
1043 
1044 	rcu_read_lock();
1045 	d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1046 	if (d)
1047 		tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1048 	else
1049 		tb_id = RT_TABLE_MAIN;
1050 	rcu_read_unlock();
1051 
1052 	return tb_id;
1053 }
1054 
1055 static struct mlxsw_sp_rif *
1056 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1057 		    const struct mlxsw_sp_rif_params *params,
1058 		    struct netlink_ext_ack *extack);
1059 
1060 static struct mlxsw_sp_rif_ipip_lb *
1061 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1062 				enum mlxsw_sp_ipip_type ipipt,
1063 				struct net_device *ol_dev,
1064 				struct netlink_ext_ack *extack)
1065 {
1066 	struct mlxsw_sp_rif_params_ipip_lb lb_params;
1067 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1068 	struct mlxsw_sp_rif *rif;
1069 
1070 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1071 	lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1072 		.common.dev = ol_dev,
1073 		.common.lag = false,
1074 		.common.double_entry = ipip_ops->double_rif_entry,
1075 		.lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1076 	};
1077 
1078 	rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1079 	if (IS_ERR(rif))
1080 		return ERR_CAST(rif);
1081 	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1082 }
1083 
1084 static struct mlxsw_sp_ipip_entry *
1085 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1086 			  enum mlxsw_sp_ipip_type ipipt,
1087 			  struct net_device *ol_dev)
1088 {
1089 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1090 	struct mlxsw_sp_ipip_entry *ipip_entry;
1091 	struct mlxsw_sp_ipip_entry *ret = NULL;
1092 	int err;
1093 
1094 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1095 	ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1096 	if (!ipip_entry)
1097 		return ERR_PTR(-ENOMEM);
1098 
1099 	ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1100 							    ol_dev, NULL);
1101 	if (IS_ERR(ipip_entry->ol_lb)) {
1102 		ret = ERR_CAST(ipip_entry->ol_lb);
1103 		goto err_ol_ipip_lb_create;
1104 	}
1105 
1106 	ipip_entry->ipipt = ipipt;
1107 	ipip_entry->ol_dev = ol_dev;
1108 	ipip_entry->parms = ipip_ops->parms_init(ol_dev);
1109 
1110 	err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry);
1111 	if (err) {
1112 		ret = ERR_PTR(err);
1113 		goto err_rem_ip_addr_set;
1114 	}
1115 
1116 	return ipip_entry;
1117 
1118 err_rem_ip_addr_set:
1119 	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1120 err_ol_ipip_lb_create:
1121 	kfree(ipip_entry);
1122 	return ret;
1123 }
1124 
1125 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp,
1126 					struct mlxsw_sp_ipip_entry *ipip_entry)
1127 {
1128 	const struct mlxsw_sp_ipip_ops *ipip_ops =
1129 		mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1130 
1131 	ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry);
1132 	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1133 	kfree(ipip_entry);
1134 }
1135 
1136 static bool
1137 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1138 				  const enum mlxsw_sp_l3proto ul_proto,
1139 				  union mlxsw_sp_l3addr saddr,
1140 				  u32 ul_tb_id,
1141 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1142 {
1143 	u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1144 	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1145 	union mlxsw_sp_l3addr tun_saddr;
1146 
1147 	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1148 		return false;
1149 
1150 	tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1151 	return tun_ul_tb_id == ul_tb_id &&
1152 	       mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1153 }
1154 
1155 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp,
1156 						 enum mlxsw_sp_ipip_type ipipt)
1157 {
1158 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1159 
1160 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1161 
1162 	/* Not all tunnels require to increase the default pasing depth
1163 	 * (96 bytes).
1164 	 */
1165 	if (ipip_ops->inc_parsing_depth)
1166 		return mlxsw_sp_parsing_depth_inc(mlxsw_sp);
1167 
1168 	return 0;
1169 }
1170 
1171 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp,
1172 						  enum mlxsw_sp_ipip_type ipipt)
1173 {
1174 	const struct mlxsw_sp_ipip_ops *ipip_ops =
1175 		mlxsw_sp->router->ipip_ops_arr[ipipt];
1176 
1177 	if (ipip_ops->inc_parsing_depth)
1178 		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
1179 }
1180 
1181 static int
1182 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1183 			      struct mlxsw_sp_fib_entry *fib_entry,
1184 			      struct mlxsw_sp_ipip_entry *ipip_entry)
1185 {
1186 	u32 tunnel_index;
1187 	int err;
1188 
1189 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1190 				  1, &tunnel_index);
1191 	if (err)
1192 		return err;
1193 
1194 	err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp,
1195 						    ipip_entry->ipipt);
1196 	if (err)
1197 		goto err_parsing_depth_inc;
1198 
1199 	ipip_entry->decap_fib_entry = fib_entry;
1200 	fib_entry->decap.ipip_entry = ipip_entry;
1201 	fib_entry->decap.tunnel_index = tunnel_index;
1202 
1203 	return 0;
1204 
1205 err_parsing_depth_inc:
1206 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
1207 			   fib_entry->decap.tunnel_index);
1208 	return err;
1209 }
1210 
1211 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1212 					  struct mlxsw_sp_fib_entry *fib_entry)
1213 {
1214 	enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt;
1215 
1216 	/* Unlink this node from the IPIP entry that it's the decap entry of. */
1217 	fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1218 	fib_entry->decap.ipip_entry = NULL;
1219 	mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt);
1220 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1221 			   1, fib_entry->decap.tunnel_index);
1222 }
1223 
1224 static struct mlxsw_sp_fib_node *
1225 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1226 			 size_t addr_len, unsigned char prefix_len);
1227 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1228 				     struct mlxsw_sp_fib_entry *fib_entry);
1229 
1230 static void
1231 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1232 				 struct mlxsw_sp_ipip_entry *ipip_entry)
1233 {
1234 	struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1235 
1236 	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1237 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1238 
1239 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1240 }
1241 
1242 static void
1243 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1244 				  struct mlxsw_sp_ipip_entry *ipip_entry,
1245 				  struct mlxsw_sp_fib_entry *decap_fib_entry)
1246 {
1247 	if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1248 					  ipip_entry))
1249 		return;
1250 	decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1251 
1252 	if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1253 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1254 }
1255 
1256 static struct mlxsw_sp_fib_entry *
1257 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1258 				     enum mlxsw_sp_l3proto proto,
1259 				     const union mlxsw_sp_l3addr *addr,
1260 				     enum mlxsw_sp_fib_entry_type type)
1261 {
1262 	struct mlxsw_sp_fib_node *fib_node;
1263 	unsigned char addr_prefix_len;
1264 	struct mlxsw_sp_fib *fib;
1265 	struct mlxsw_sp_vr *vr;
1266 	const void *addrp;
1267 	size_t addr_len;
1268 	u32 addr4;
1269 
1270 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1271 	if (!vr)
1272 		return NULL;
1273 	fib = mlxsw_sp_vr_fib(vr, proto);
1274 
1275 	switch (proto) {
1276 	case MLXSW_SP_L3_PROTO_IPV4:
1277 		addr4 = be32_to_cpu(addr->addr4);
1278 		addrp = &addr4;
1279 		addr_len = 4;
1280 		addr_prefix_len = 32;
1281 		break;
1282 	case MLXSW_SP_L3_PROTO_IPV6:
1283 		addrp = &addr->addr6;
1284 		addr_len = 16;
1285 		addr_prefix_len = 128;
1286 		break;
1287 	default:
1288 		WARN_ON(1);
1289 		return NULL;
1290 	}
1291 
1292 	fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1293 					    addr_prefix_len);
1294 	if (!fib_node || fib_node->fib_entry->type != type)
1295 		return NULL;
1296 
1297 	return fib_node->fib_entry;
1298 }
1299 
1300 /* Given an IPIP entry, find the corresponding decap route. */
1301 static struct mlxsw_sp_fib_entry *
1302 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1303 			       struct mlxsw_sp_ipip_entry *ipip_entry)
1304 {
1305 	static struct mlxsw_sp_fib_node *fib_node;
1306 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1307 	unsigned char saddr_prefix_len;
1308 	union mlxsw_sp_l3addr saddr;
1309 	struct mlxsw_sp_fib *ul_fib;
1310 	struct mlxsw_sp_vr *ul_vr;
1311 	const void *saddrp;
1312 	size_t saddr_len;
1313 	u32 ul_tb_id;
1314 	u32 saddr4;
1315 
1316 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1317 
1318 	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1319 	ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1320 	if (!ul_vr)
1321 		return NULL;
1322 
1323 	ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1324 	saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1325 					   ipip_entry->ol_dev);
1326 
1327 	switch (ipip_ops->ul_proto) {
1328 	case MLXSW_SP_L3_PROTO_IPV4:
1329 		saddr4 = be32_to_cpu(saddr.addr4);
1330 		saddrp = &saddr4;
1331 		saddr_len = 4;
1332 		saddr_prefix_len = 32;
1333 		break;
1334 	case MLXSW_SP_L3_PROTO_IPV6:
1335 		saddrp = &saddr.addr6;
1336 		saddr_len = 16;
1337 		saddr_prefix_len = 128;
1338 		break;
1339 	default:
1340 		WARN_ON(1);
1341 		return NULL;
1342 	}
1343 
1344 	fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1345 					    saddr_prefix_len);
1346 	if (!fib_node ||
1347 	    fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1348 		return NULL;
1349 
1350 	return fib_node->fib_entry;
1351 }
1352 
1353 static struct mlxsw_sp_ipip_entry *
1354 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1355 			   enum mlxsw_sp_ipip_type ipipt,
1356 			   struct net_device *ol_dev)
1357 {
1358 	struct mlxsw_sp_ipip_entry *ipip_entry;
1359 
1360 	ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1361 	if (IS_ERR(ipip_entry))
1362 		return ipip_entry;
1363 
1364 	list_add_tail(&ipip_entry->ipip_list_node,
1365 		      &mlxsw_sp->router->ipip_list);
1366 
1367 	return ipip_entry;
1368 }
1369 
1370 static void
1371 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1372 			    struct mlxsw_sp_ipip_entry *ipip_entry)
1373 {
1374 	list_del(&ipip_entry->ipip_list_node);
1375 	mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry);
1376 }
1377 
1378 static bool
1379 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1380 				  const struct net_device *ul_dev,
1381 				  enum mlxsw_sp_l3proto ul_proto,
1382 				  union mlxsw_sp_l3addr ul_dip,
1383 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1384 {
1385 	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1386 	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1387 
1388 	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1389 		return false;
1390 
1391 	return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1392 						 ul_tb_id, ipip_entry);
1393 }
1394 
1395 /* Given decap parameters, find the corresponding IPIP entry. */
1396 static struct mlxsw_sp_ipip_entry *
1397 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1398 				  enum mlxsw_sp_l3proto ul_proto,
1399 				  union mlxsw_sp_l3addr ul_dip)
1400 {
1401 	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1402 	struct net_device *ul_dev;
1403 
1404 	rcu_read_lock();
1405 
1406 	ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1407 	if (!ul_dev)
1408 		goto out_unlock;
1409 
1410 	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1411 			    ipip_list_node)
1412 		if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1413 						      ul_proto, ul_dip,
1414 						      ipip_entry))
1415 			goto out_unlock;
1416 
1417 	rcu_read_unlock();
1418 
1419 	return NULL;
1420 
1421 out_unlock:
1422 	rcu_read_unlock();
1423 	return ipip_entry;
1424 }
1425 
1426 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1427 				      const struct net_device *dev,
1428 				      enum mlxsw_sp_ipip_type *p_type)
1429 {
1430 	struct mlxsw_sp_router *router = mlxsw_sp->router;
1431 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1432 	enum mlxsw_sp_ipip_type ipipt;
1433 
1434 	for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1435 		ipip_ops = router->ipip_ops_arr[ipipt];
1436 		if (dev->type == ipip_ops->dev_type) {
1437 			if (p_type)
1438 				*p_type = ipipt;
1439 			return true;
1440 		}
1441 	}
1442 	return false;
1443 }
1444 
1445 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1446 				       const struct net_device *dev)
1447 {
1448 	return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1449 }
1450 
1451 static struct mlxsw_sp_ipip_entry *
1452 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1453 				   const struct net_device *ol_dev)
1454 {
1455 	struct mlxsw_sp_ipip_entry *ipip_entry;
1456 
1457 	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1458 			    ipip_list_node)
1459 		if (ipip_entry->ol_dev == ol_dev)
1460 			return ipip_entry;
1461 
1462 	return NULL;
1463 }
1464 
1465 static struct mlxsw_sp_ipip_entry *
1466 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1467 				   const struct net_device *ul_dev,
1468 				   struct mlxsw_sp_ipip_entry *start)
1469 {
1470 	struct mlxsw_sp_ipip_entry *ipip_entry;
1471 
1472 	ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1473 					ipip_list_node);
1474 	list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1475 				     ipip_list_node) {
1476 		struct net_device *ol_dev = ipip_entry->ol_dev;
1477 		struct net_device *ipip_ul_dev;
1478 
1479 		rcu_read_lock();
1480 		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1481 		rcu_read_unlock();
1482 
1483 		if (ipip_ul_dev == ul_dev)
1484 			return ipip_entry;
1485 	}
1486 
1487 	return NULL;
1488 }
1489 
1490 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1491 				       const struct net_device *dev)
1492 {
1493 	return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1494 }
1495 
1496 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1497 						const struct net_device *ol_dev,
1498 						enum mlxsw_sp_ipip_type ipipt)
1499 {
1500 	const struct mlxsw_sp_ipip_ops *ops
1501 		= mlxsw_sp->router->ipip_ops_arr[ipipt];
1502 
1503 	return ops->can_offload(mlxsw_sp, ol_dev);
1504 }
1505 
1506 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1507 						struct net_device *ol_dev)
1508 {
1509 	enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1510 	struct mlxsw_sp_ipip_entry *ipip_entry;
1511 	enum mlxsw_sp_l3proto ul_proto;
1512 	union mlxsw_sp_l3addr saddr;
1513 	u32 ul_tb_id;
1514 
1515 	mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1516 	if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1517 		ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1518 		ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1519 		saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1520 		if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1521 							  saddr, ul_tb_id,
1522 							  NULL)) {
1523 			ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1524 								ol_dev);
1525 			if (IS_ERR(ipip_entry))
1526 				return PTR_ERR(ipip_entry);
1527 		}
1528 	}
1529 
1530 	return 0;
1531 }
1532 
1533 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1534 						   struct net_device *ol_dev)
1535 {
1536 	struct mlxsw_sp_ipip_entry *ipip_entry;
1537 
1538 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1539 	if (ipip_entry)
1540 		mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1541 }
1542 
1543 static void
1544 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1545 				struct mlxsw_sp_ipip_entry *ipip_entry)
1546 {
1547 	struct mlxsw_sp_fib_entry *decap_fib_entry;
1548 
1549 	decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1550 	if (decap_fib_entry)
1551 		mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1552 						  decap_fib_entry);
1553 }
1554 
1555 static int
1556 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1557 			u16 ul_rif_id, bool enable)
1558 {
1559 	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1560 	enum mlxsw_reg_ritr_loopback_ipip_options ipip_options;
1561 	struct mlxsw_sp_rif *rif = &lb_rif->common;
1562 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1563 	char ritr_pl[MLXSW_REG_RITR_LEN];
1564 	struct in6_addr *saddr6;
1565 	u32 saddr4;
1566 
1567 	ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET;
1568 	switch (lb_cf.ul_protocol) {
1569 	case MLXSW_SP_L3_PROTO_IPV4:
1570 		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1571 		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1572 				    rif->rif_index, rif->vr_id, rif->dev->mtu);
1573 		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1574 						   ipip_options, ul_vr_id,
1575 						   ul_rif_id, saddr4,
1576 						   lb_cf.okey);
1577 		break;
1578 
1579 	case MLXSW_SP_L3_PROTO_IPV6:
1580 		saddr6 = &lb_cf.saddr.addr6;
1581 		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1582 				    rif->rif_index, rif->vr_id, rif->dev->mtu);
1583 		mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt,
1584 						   ipip_options, ul_vr_id,
1585 						   ul_rif_id, saddr6,
1586 						   lb_cf.okey);
1587 		break;
1588 	}
1589 
1590 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1591 }
1592 
1593 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1594 						 struct net_device *ol_dev)
1595 {
1596 	struct mlxsw_sp_ipip_entry *ipip_entry;
1597 	struct mlxsw_sp_rif_ipip_lb *lb_rif;
1598 	int err = 0;
1599 
1600 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1601 	if (ipip_entry) {
1602 		lb_rif = ipip_entry->ol_lb;
1603 		err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1604 					      lb_rif->ul_rif_id, true);
1605 		if (err)
1606 			goto out;
1607 		lb_rif->common.mtu = ol_dev->mtu;
1608 	}
1609 
1610 out:
1611 	return err;
1612 }
1613 
1614 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1615 						struct net_device *ol_dev)
1616 {
1617 	struct mlxsw_sp_ipip_entry *ipip_entry;
1618 
1619 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1620 	if (ipip_entry)
1621 		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1622 }
1623 
1624 static void
1625 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1626 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1627 {
1628 	if (ipip_entry->decap_fib_entry)
1629 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1630 }
1631 
1632 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1633 						  struct net_device *ol_dev)
1634 {
1635 	struct mlxsw_sp_ipip_entry *ipip_entry;
1636 
1637 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1638 	if (ipip_entry)
1639 		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1640 }
1641 
1642 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1643 					 struct mlxsw_sp_rif *old_rif,
1644 					 struct mlxsw_sp_rif *new_rif);
1645 static int
1646 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1647 				 struct mlxsw_sp_ipip_entry *ipip_entry,
1648 				 bool keep_encap,
1649 				 struct netlink_ext_ack *extack)
1650 {
1651 	struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1652 	struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1653 
1654 	new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1655 						     ipip_entry->ipipt,
1656 						     ipip_entry->ol_dev,
1657 						     extack);
1658 	if (IS_ERR(new_lb_rif))
1659 		return PTR_ERR(new_lb_rif);
1660 	ipip_entry->ol_lb = new_lb_rif;
1661 
1662 	if (keep_encap)
1663 		mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1664 					     &new_lb_rif->common);
1665 
1666 	mlxsw_sp_rif_destroy(&old_lb_rif->common);
1667 
1668 	return 0;
1669 }
1670 
1671 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1672 					struct mlxsw_sp_rif *rif);
1673 
1674 /**
1675  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1676  * @mlxsw_sp: mlxsw_sp.
1677  * @ipip_entry: IPIP entry.
1678  * @recreate_loopback: Recreates the associated loopback RIF.
1679  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1680  *              relevant when recreate_loopback is true.
1681  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1682  *                   is only relevant when recreate_loopback is false.
1683  * @extack: extack.
1684  *
1685  * Return: Non-zero value on failure.
1686  */
1687 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1688 					struct mlxsw_sp_ipip_entry *ipip_entry,
1689 					bool recreate_loopback,
1690 					bool keep_encap,
1691 					bool update_nexthops,
1692 					struct netlink_ext_ack *extack)
1693 {
1694 	int err;
1695 
1696 	/* RIFs can't be edited, so to update loopback, we need to destroy and
1697 	 * recreate it. That creates a window of opportunity where RALUE and
1698 	 * RATR registers end up referencing a RIF that's already gone. RATRs
1699 	 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1700 	 * of RALUE, demote the decap route back.
1701 	 */
1702 	if (ipip_entry->decap_fib_entry)
1703 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1704 
1705 	if (recreate_loopback) {
1706 		err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1707 						       keep_encap, extack);
1708 		if (err)
1709 			return err;
1710 	} else if (update_nexthops) {
1711 		mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1712 					    &ipip_entry->ol_lb->common);
1713 	}
1714 
1715 	if (ipip_entry->ol_dev->flags & IFF_UP)
1716 		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1717 
1718 	return 0;
1719 }
1720 
1721 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1722 						struct net_device *ol_dev,
1723 						struct netlink_ext_ack *extack)
1724 {
1725 	struct mlxsw_sp_ipip_entry *ipip_entry =
1726 		mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1727 
1728 	if (!ipip_entry)
1729 		return 0;
1730 
1731 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1732 						   true, false, false, extack);
1733 }
1734 
1735 static int
1736 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1737 				     struct mlxsw_sp_ipip_entry *ipip_entry,
1738 				     struct net_device *ul_dev,
1739 				     bool *demote_this,
1740 				     struct netlink_ext_ack *extack)
1741 {
1742 	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1743 	enum mlxsw_sp_l3proto ul_proto;
1744 	union mlxsw_sp_l3addr saddr;
1745 
1746 	/* Moving underlay to a different VRF might cause local address
1747 	 * conflict, and the conflicting tunnels need to be demoted.
1748 	 */
1749 	ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1750 	saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1751 	if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1752 						 saddr, ul_tb_id,
1753 						 ipip_entry)) {
1754 		*demote_this = true;
1755 		return 0;
1756 	}
1757 
1758 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1759 						   true, true, false, extack);
1760 }
1761 
1762 static int
1763 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1764 				    struct mlxsw_sp_ipip_entry *ipip_entry,
1765 				    struct net_device *ul_dev)
1766 {
1767 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1768 						   false, false, true, NULL);
1769 }
1770 
1771 static int
1772 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1773 				      struct mlxsw_sp_ipip_entry *ipip_entry,
1774 				      struct net_device *ul_dev)
1775 {
1776 	/* A down underlay device causes encapsulated packets to not be
1777 	 * forwarded, but decap still works. So refresh next hops without
1778 	 * touching anything else.
1779 	 */
1780 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1781 						   false, false, true, NULL);
1782 }
1783 
1784 static int
1785 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1786 					struct net_device *ol_dev,
1787 					struct netlink_ext_ack *extack)
1788 {
1789 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1790 	struct mlxsw_sp_ipip_entry *ipip_entry;
1791 	int err;
1792 
1793 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1794 	if (!ipip_entry)
1795 		/* A change might make a tunnel eligible for offloading, but
1796 		 * that is currently not implemented. What falls to slow path
1797 		 * stays there.
1798 		 */
1799 		return 0;
1800 
1801 	/* A change might make a tunnel not eligible for offloading. */
1802 	if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1803 						 ipip_entry->ipipt)) {
1804 		mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1805 		return 0;
1806 	}
1807 
1808 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1809 	err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1810 	return err;
1811 }
1812 
1813 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1814 				       struct mlxsw_sp_ipip_entry *ipip_entry)
1815 {
1816 	struct net_device *ol_dev = ipip_entry->ol_dev;
1817 
1818 	if (ol_dev->flags & IFF_UP)
1819 		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1820 	mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1821 }
1822 
1823 /* The configuration where several tunnels have the same local address in the
1824  * same underlay table needs special treatment in the HW. That is currently not
1825  * implemented in the driver. This function finds and demotes the first tunnel
1826  * with a given source address, except the one passed in the argument
1827  * `except'.
1828  */
1829 bool
1830 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1831 				     enum mlxsw_sp_l3proto ul_proto,
1832 				     union mlxsw_sp_l3addr saddr,
1833 				     u32 ul_tb_id,
1834 				     const struct mlxsw_sp_ipip_entry *except)
1835 {
1836 	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1837 
1838 	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1839 				 ipip_list_node) {
1840 		if (ipip_entry != except &&
1841 		    mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1842 						      ul_tb_id, ipip_entry)) {
1843 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1844 			return true;
1845 		}
1846 	}
1847 
1848 	return false;
1849 }
1850 
1851 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1852 						     struct net_device *ul_dev)
1853 {
1854 	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1855 
1856 	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1857 				 ipip_list_node) {
1858 		struct net_device *ol_dev = ipip_entry->ol_dev;
1859 		struct net_device *ipip_ul_dev;
1860 
1861 		rcu_read_lock();
1862 		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1863 		rcu_read_unlock();
1864 		if (ipip_ul_dev == ul_dev)
1865 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1866 	}
1867 }
1868 
1869 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1870 					    struct net_device *ol_dev,
1871 					    unsigned long event,
1872 					    struct netdev_notifier_info *info)
1873 {
1874 	struct netdev_notifier_changeupper_info *chup;
1875 	struct netlink_ext_ack *extack;
1876 	int err = 0;
1877 
1878 	switch (event) {
1879 	case NETDEV_REGISTER:
1880 		err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1881 		break;
1882 	case NETDEV_UNREGISTER:
1883 		mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1884 		break;
1885 	case NETDEV_UP:
1886 		mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1887 		break;
1888 	case NETDEV_DOWN:
1889 		mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1890 		break;
1891 	case NETDEV_CHANGEUPPER:
1892 		chup = container_of(info, typeof(*chup), info);
1893 		extack = info->extack;
1894 		if (netif_is_l3_master(chup->upper_dev))
1895 			err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1896 								   ol_dev,
1897 								   extack);
1898 		break;
1899 	case NETDEV_CHANGE:
1900 		extack = info->extack;
1901 		err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1902 							      ol_dev, extack);
1903 		break;
1904 	case NETDEV_CHANGEMTU:
1905 		err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1906 		break;
1907 	}
1908 	return err;
1909 }
1910 
1911 static int
1912 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1913 				   struct mlxsw_sp_ipip_entry *ipip_entry,
1914 				   struct net_device *ul_dev,
1915 				   bool *demote_this,
1916 				   unsigned long event,
1917 				   struct netdev_notifier_info *info)
1918 {
1919 	struct netdev_notifier_changeupper_info *chup;
1920 	struct netlink_ext_ack *extack;
1921 
1922 	switch (event) {
1923 	case NETDEV_CHANGEUPPER:
1924 		chup = container_of(info, typeof(*chup), info);
1925 		extack = info->extack;
1926 		if (netif_is_l3_master(chup->upper_dev))
1927 			return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1928 								    ipip_entry,
1929 								    ul_dev,
1930 								    demote_this,
1931 								    extack);
1932 		break;
1933 
1934 	case NETDEV_UP:
1935 		return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1936 							   ul_dev);
1937 	case NETDEV_DOWN:
1938 		return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1939 							     ipip_entry,
1940 							     ul_dev);
1941 	}
1942 	return 0;
1943 }
1944 
1945 static int
1946 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1947 				 struct net_device *ul_dev,
1948 				 unsigned long event,
1949 				 struct netdev_notifier_info *info)
1950 {
1951 	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1952 	int err;
1953 
1954 	while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1955 								ul_dev,
1956 								ipip_entry))) {
1957 		struct mlxsw_sp_ipip_entry *prev;
1958 		bool demote_this = false;
1959 
1960 		err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1961 							 ul_dev, &demote_this,
1962 							 event, info);
1963 		if (err) {
1964 			mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1965 								 ul_dev);
1966 			return err;
1967 		}
1968 
1969 		if (demote_this) {
1970 			if (list_is_first(&ipip_entry->ipip_list_node,
1971 					  &mlxsw_sp->router->ipip_list))
1972 				prev = NULL;
1973 			else
1974 				/* This can't be cached from previous iteration,
1975 				 * because that entry could be gone now.
1976 				 */
1977 				prev = list_prev_entry(ipip_entry,
1978 						       ipip_list_node);
1979 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1980 			ipip_entry = prev;
1981 		}
1982 	}
1983 
1984 	return 0;
1985 }
1986 
1987 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1988 				      enum mlxsw_sp_l3proto ul_proto,
1989 				      const union mlxsw_sp_l3addr *ul_sip,
1990 				      u32 tunnel_index)
1991 {
1992 	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1993 	struct mlxsw_sp_router *router = mlxsw_sp->router;
1994 	struct mlxsw_sp_fib_entry *fib_entry;
1995 	int err = 0;
1996 
1997 	mutex_lock(&mlxsw_sp->router->lock);
1998 
1999 	if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
2000 		err = -EINVAL;
2001 		goto out;
2002 	}
2003 
2004 	router->nve_decap_config.ul_tb_id = ul_tb_id;
2005 	router->nve_decap_config.tunnel_index = tunnel_index;
2006 	router->nve_decap_config.ul_proto = ul_proto;
2007 	router->nve_decap_config.ul_sip = *ul_sip;
2008 	router->nve_decap_config.valid = true;
2009 
2010 	/* It is valid to create a tunnel with a local IP and only later
2011 	 * assign this IP address to a local interface
2012 	 */
2013 	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2014 							 ul_proto, ul_sip,
2015 							 type);
2016 	if (!fib_entry)
2017 		goto out;
2018 
2019 	fib_entry->decap.tunnel_index = tunnel_index;
2020 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2021 
2022 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2023 	if (err)
2024 		goto err_fib_entry_update;
2025 
2026 	goto out;
2027 
2028 err_fib_entry_update:
2029 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2030 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2031 out:
2032 	mutex_unlock(&mlxsw_sp->router->lock);
2033 	return err;
2034 }
2035 
2036 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2037 				      enum mlxsw_sp_l3proto ul_proto,
2038 				      const union mlxsw_sp_l3addr *ul_sip)
2039 {
2040 	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2041 	struct mlxsw_sp_router *router = mlxsw_sp->router;
2042 	struct mlxsw_sp_fib_entry *fib_entry;
2043 
2044 	mutex_lock(&mlxsw_sp->router->lock);
2045 
2046 	if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2047 		goto out;
2048 
2049 	router->nve_decap_config.valid = false;
2050 
2051 	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2052 							 ul_proto, ul_sip,
2053 							 type);
2054 	if (!fib_entry)
2055 		goto out;
2056 
2057 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2058 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2059 out:
2060 	mutex_unlock(&mlxsw_sp->router->lock);
2061 }
2062 
2063 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2064 					 u32 ul_tb_id,
2065 					 enum mlxsw_sp_l3proto ul_proto,
2066 					 const union mlxsw_sp_l3addr *ul_sip)
2067 {
2068 	struct mlxsw_sp_router *router = mlxsw_sp->router;
2069 
2070 	return router->nve_decap_config.valid &&
2071 	       router->nve_decap_config.ul_tb_id == ul_tb_id &&
2072 	       router->nve_decap_config.ul_proto == ul_proto &&
2073 	       !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2074 		       sizeof(*ul_sip));
2075 }
2076 
2077 struct mlxsw_sp_neigh_key {
2078 	struct neighbour *n;
2079 };
2080 
2081 struct mlxsw_sp_neigh_entry {
2082 	struct list_head rif_list_node;
2083 	struct rhash_head ht_node;
2084 	struct mlxsw_sp_neigh_key key;
2085 	u16 rif;
2086 	bool connected;
2087 	unsigned char ha[ETH_ALEN];
2088 	struct list_head nexthop_list; /* list of nexthops using
2089 					* this neigh entry
2090 					*/
2091 	struct list_head nexthop_neighs_list_node;
2092 	unsigned int counter_index;
2093 	bool counter_valid;
2094 };
2095 
2096 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2097 	.key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2098 	.head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2099 	.key_len = sizeof(struct mlxsw_sp_neigh_key),
2100 };
2101 
2102 struct mlxsw_sp_neigh_entry *
2103 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2104 			struct mlxsw_sp_neigh_entry *neigh_entry)
2105 {
2106 	if (!neigh_entry) {
2107 		if (list_empty(&rif->neigh_list))
2108 			return NULL;
2109 		else
2110 			return list_first_entry(&rif->neigh_list,
2111 						typeof(*neigh_entry),
2112 						rif_list_node);
2113 	}
2114 	if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2115 		return NULL;
2116 	return list_next_entry(neigh_entry, rif_list_node);
2117 }
2118 
2119 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2120 {
2121 	return neigh_entry->key.n->tbl->family;
2122 }
2123 
2124 unsigned char *
2125 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2126 {
2127 	return neigh_entry->ha;
2128 }
2129 
2130 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2131 {
2132 	struct neighbour *n;
2133 
2134 	n = neigh_entry->key.n;
2135 	return ntohl(*((__be32 *) n->primary_key));
2136 }
2137 
2138 struct in6_addr *
2139 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2140 {
2141 	struct neighbour *n;
2142 
2143 	n = neigh_entry->key.n;
2144 	return (struct in6_addr *) &n->primary_key;
2145 }
2146 
2147 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2148 			       struct mlxsw_sp_neigh_entry *neigh_entry,
2149 			       u64 *p_counter)
2150 {
2151 	if (!neigh_entry->counter_valid)
2152 		return -EINVAL;
2153 
2154 	return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2155 					 p_counter, NULL);
2156 }
2157 
2158 static struct mlxsw_sp_neigh_entry *
2159 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2160 			   u16 rif)
2161 {
2162 	struct mlxsw_sp_neigh_entry *neigh_entry;
2163 
2164 	neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2165 	if (!neigh_entry)
2166 		return NULL;
2167 
2168 	neigh_entry->key.n = n;
2169 	neigh_entry->rif = rif;
2170 	INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2171 
2172 	return neigh_entry;
2173 }
2174 
2175 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2176 {
2177 	kfree(neigh_entry);
2178 }
2179 
2180 static int
2181 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2182 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2183 {
2184 	return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2185 				      &neigh_entry->ht_node,
2186 				      mlxsw_sp_neigh_ht_params);
2187 }
2188 
2189 static void
2190 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2191 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2192 {
2193 	rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2194 			       &neigh_entry->ht_node,
2195 			       mlxsw_sp_neigh_ht_params);
2196 }
2197 
2198 static bool
2199 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2200 				    struct mlxsw_sp_neigh_entry *neigh_entry)
2201 {
2202 	struct devlink *devlink;
2203 	const char *table_name;
2204 
2205 	switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2206 	case AF_INET:
2207 		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2208 		break;
2209 	case AF_INET6:
2210 		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2211 		break;
2212 	default:
2213 		WARN_ON(1);
2214 		return false;
2215 	}
2216 
2217 	devlink = priv_to_devlink(mlxsw_sp->core);
2218 	return devlink_dpipe_table_counter_enabled(devlink, table_name);
2219 }
2220 
2221 static void
2222 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2223 			     struct mlxsw_sp_neigh_entry *neigh_entry)
2224 {
2225 	if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2226 		return;
2227 
2228 	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2229 		return;
2230 
2231 	neigh_entry->counter_valid = true;
2232 }
2233 
2234 static void
2235 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2236 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2237 {
2238 	if (!neigh_entry->counter_valid)
2239 		return;
2240 	mlxsw_sp_flow_counter_free(mlxsw_sp,
2241 				   neigh_entry->counter_index);
2242 	neigh_entry->counter_valid = false;
2243 }
2244 
2245 static struct mlxsw_sp_neigh_entry *
2246 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2247 {
2248 	struct mlxsw_sp_neigh_entry *neigh_entry;
2249 	struct mlxsw_sp_rif *rif;
2250 	int err;
2251 
2252 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2253 	if (!rif)
2254 		return ERR_PTR(-EINVAL);
2255 
2256 	neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2257 	if (!neigh_entry)
2258 		return ERR_PTR(-ENOMEM);
2259 
2260 	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2261 	if (err)
2262 		goto err_neigh_entry_insert;
2263 
2264 	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2265 	atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count);
2266 	list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2267 
2268 	return neigh_entry;
2269 
2270 err_neigh_entry_insert:
2271 	mlxsw_sp_neigh_entry_free(neigh_entry);
2272 	return ERR_PTR(err);
2273 }
2274 
2275 static void
2276 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2277 			     struct mlxsw_sp_neigh_entry *neigh_entry)
2278 {
2279 	list_del(&neigh_entry->rif_list_node);
2280 	atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count);
2281 	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2282 	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2283 	mlxsw_sp_neigh_entry_free(neigh_entry);
2284 }
2285 
2286 static struct mlxsw_sp_neigh_entry *
2287 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2288 {
2289 	struct mlxsw_sp_neigh_key key;
2290 
2291 	key.n = n;
2292 	return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2293 				      &key, mlxsw_sp_neigh_ht_params);
2294 }
2295 
2296 static void
2297 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2298 {
2299 	unsigned long interval;
2300 
2301 #if IS_ENABLED(CONFIG_IPV6)
2302 	interval = min_t(unsigned long,
2303 			 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2304 			 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2305 #else
2306 	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2307 #endif
2308 	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2309 }
2310 
2311 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2312 						   char *rauhtd_pl,
2313 						   int ent_index)
2314 {
2315 	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2316 	struct net_device *dev;
2317 	struct neighbour *n;
2318 	__be32 dipn;
2319 	u32 dip;
2320 	u16 rif;
2321 
2322 	mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2323 
2324 	if (WARN_ON_ONCE(rif >= max_rifs))
2325 		return;
2326 	if (!mlxsw_sp->router->rifs[rif]) {
2327 		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2328 		return;
2329 	}
2330 
2331 	dipn = htonl(dip);
2332 	dev = mlxsw_sp->router->rifs[rif]->dev;
2333 	n = neigh_lookup(&arp_tbl, &dipn, dev);
2334 	if (!n)
2335 		return;
2336 
2337 	netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2338 	neigh_event_send(n, NULL);
2339 	neigh_release(n);
2340 }
2341 
2342 #if IS_ENABLED(CONFIG_IPV6)
2343 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2344 						   char *rauhtd_pl,
2345 						   int rec_index)
2346 {
2347 	struct net_device *dev;
2348 	struct neighbour *n;
2349 	struct in6_addr dip;
2350 	u16 rif;
2351 
2352 	mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2353 					 (char *) &dip);
2354 
2355 	if (!mlxsw_sp->router->rifs[rif]) {
2356 		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2357 		return;
2358 	}
2359 
2360 	dev = mlxsw_sp->router->rifs[rif]->dev;
2361 	n = neigh_lookup(&nd_tbl, &dip, dev);
2362 	if (!n)
2363 		return;
2364 
2365 	netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2366 	neigh_event_send(n, NULL);
2367 	neigh_release(n);
2368 }
2369 #else
2370 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2371 						   char *rauhtd_pl,
2372 						   int rec_index)
2373 {
2374 }
2375 #endif
2376 
2377 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2378 						   char *rauhtd_pl,
2379 						   int rec_index)
2380 {
2381 	u8 num_entries;
2382 	int i;
2383 
2384 	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2385 								rec_index);
2386 	/* Hardware starts counting at 0, so add 1. */
2387 	num_entries++;
2388 
2389 	/* Each record consists of several neighbour entries. */
2390 	for (i = 0; i < num_entries; i++) {
2391 		int ent_index;
2392 
2393 		ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2394 		mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2395 						       ent_index);
2396 	}
2397 
2398 }
2399 
2400 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2401 						   char *rauhtd_pl,
2402 						   int rec_index)
2403 {
2404 	/* One record contains one entry. */
2405 	mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2406 					       rec_index);
2407 }
2408 
2409 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2410 					      char *rauhtd_pl, int rec_index)
2411 {
2412 	switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2413 	case MLXSW_REG_RAUHTD_TYPE_IPV4:
2414 		mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2415 						       rec_index);
2416 		break;
2417 	case MLXSW_REG_RAUHTD_TYPE_IPV6:
2418 		mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2419 						       rec_index);
2420 		break;
2421 	}
2422 }
2423 
2424 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2425 {
2426 	u8 num_rec, last_rec_index, num_entries;
2427 
2428 	num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2429 	last_rec_index = num_rec - 1;
2430 
2431 	if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2432 		return false;
2433 	if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2434 	    MLXSW_REG_RAUHTD_TYPE_IPV6)
2435 		return true;
2436 
2437 	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2438 								last_rec_index);
2439 	if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2440 		return true;
2441 	return false;
2442 }
2443 
2444 static int
2445 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2446 				       char *rauhtd_pl,
2447 				       enum mlxsw_reg_rauhtd_type type)
2448 {
2449 	int i, num_rec;
2450 	int err;
2451 
2452 	/* Ensure the RIF we read from the device does not change mid-dump. */
2453 	mutex_lock(&mlxsw_sp->router->lock);
2454 	do {
2455 		mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2456 		err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2457 				      rauhtd_pl);
2458 		if (err) {
2459 			dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2460 			break;
2461 		}
2462 		num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2463 		for (i = 0; i < num_rec; i++)
2464 			mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2465 							  i);
2466 	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2467 	mutex_unlock(&mlxsw_sp->router->lock);
2468 
2469 	return err;
2470 }
2471 
2472 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2473 {
2474 	enum mlxsw_reg_rauhtd_type type;
2475 	char *rauhtd_pl;
2476 	int err;
2477 
2478 	if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count))
2479 		return 0;
2480 
2481 	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2482 	if (!rauhtd_pl)
2483 		return -ENOMEM;
2484 
2485 	type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2486 	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2487 	if (err)
2488 		goto out;
2489 
2490 	type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2491 	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2492 out:
2493 	kfree(rauhtd_pl);
2494 	return err;
2495 }
2496 
2497 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2498 {
2499 	struct mlxsw_sp_neigh_entry *neigh_entry;
2500 
2501 	mutex_lock(&mlxsw_sp->router->lock);
2502 	list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2503 			    nexthop_neighs_list_node)
2504 		/* If this neigh have nexthops, make the kernel think this neigh
2505 		 * is active regardless of the traffic.
2506 		 */
2507 		neigh_event_send(neigh_entry->key.n, NULL);
2508 	mutex_unlock(&mlxsw_sp->router->lock);
2509 }
2510 
2511 static void
2512 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2513 {
2514 	unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2515 
2516 	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2517 			       msecs_to_jiffies(interval));
2518 }
2519 
2520 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2521 {
2522 	struct mlxsw_sp_router *router;
2523 	int err;
2524 
2525 	router = container_of(work, struct mlxsw_sp_router,
2526 			      neighs_update.dw.work);
2527 	err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2528 	if (err)
2529 		dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2530 
2531 	mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2532 
2533 	mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2534 }
2535 
2536 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2537 {
2538 	struct mlxsw_sp_neigh_entry *neigh_entry;
2539 	struct mlxsw_sp_router *router;
2540 
2541 	router = container_of(work, struct mlxsw_sp_router,
2542 			      nexthop_probe_dw.work);
2543 	/* Iterate over nexthop neighbours, find those who are unresolved and
2544 	 * send arp on them. This solves the chicken-egg problem when
2545 	 * the nexthop wouldn't get offloaded until the neighbor is resolved
2546 	 * but it wouldn't get resolved ever in case traffic is flowing in HW
2547 	 * using different nexthop.
2548 	 */
2549 	mutex_lock(&router->lock);
2550 	list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2551 			    nexthop_neighs_list_node)
2552 		if (!neigh_entry->connected)
2553 			neigh_event_send(neigh_entry->key.n, NULL);
2554 	mutex_unlock(&router->lock);
2555 
2556 	mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2557 			       MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2558 }
2559 
2560 static void
2561 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2562 			      struct mlxsw_sp_neigh_entry *neigh_entry,
2563 			      bool removing, bool dead);
2564 
2565 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2566 {
2567 	return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2568 			MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2569 }
2570 
2571 static int
2572 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2573 				struct mlxsw_sp_neigh_entry *neigh_entry,
2574 				enum mlxsw_reg_rauht_op op)
2575 {
2576 	struct neighbour *n = neigh_entry->key.n;
2577 	u32 dip = ntohl(*((__be32 *) n->primary_key));
2578 	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2579 
2580 	mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2581 			      dip);
2582 	if (neigh_entry->counter_valid)
2583 		mlxsw_reg_rauht_pack_counter(rauht_pl,
2584 					     neigh_entry->counter_index);
2585 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2586 }
2587 
2588 static int
2589 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2590 				struct mlxsw_sp_neigh_entry *neigh_entry,
2591 				enum mlxsw_reg_rauht_op op)
2592 {
2593 	struct neighbour *n = neigh_entry->key.n;
2594 	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2595 	const char *dip = n->primary_key;
2596 
2597 	mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2598 			      dip);
2599 	if (neigh_entry->counter_valid)
2600 		mlxsw_reg_rauht_pack_counter(rauht_pl,
2601 					     neigh_entry->counter_index);
2602 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2603 }
2604 
2605 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2606 {
2607 	struct neighbour *n = neigh_entry->key.n;
2608 
2609 	/* Packets with a link-local destination address are trapped
2610 	 * after LPM lookup and never reach the neighbour table, so
2611 	 * there is no need to program such neighbours to the device.
2612 	 */
2613 	if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2614 	    IPV6_ADDR_LINKLOCAL)
2615 		return true;
2616 	return false;
2617 }
2618 
2619 static void
2620 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2621 			    struct mlxsw_sp_neigh_entry *neigh_entry,
2622 			    bool adding)
2623 {
2624 	enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2625 	int err;
2626 
2627 	if (!adding && !neigh_entry->connected)
2628 		return;
2629 	neigh_entry->connected = adding;
2630 	if (neigh_entry->key.n->tbl->family == AF_INET) {
2631 		err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2632 						      op);
2633 		if (err)
2634 			return;
2635 	} else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2636 		if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2637 			return;
2638 		err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2639 						      op);
2640 		if (err)
2641 			return;
2642 	} else {
2643 		WARN_ON_ONCE(1);
2644 		return;
2645 	}
2646 
2647 	if (adding)
2648 		neigh_entry->key.n->flags |= NTF_OFFLOADED;
2649 	else
2650 		neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2651 }
2652 
2653 void
2654 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2655 				    struct mlxsw_sp_neigh_entry *neigh_entry,
2656 				    bool adding)
2657 {
2658 	if (adding)
2659 		mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2660 	else
2661 		mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2662 	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2663 }
2664 
2665 struct mlxsw_sp_netevent_work {
2666 	struct work_struct work;
2667 	struct mlxsw_sp *mlxsw_sp;
2668 	struct neighbour *n;
2669 };
2670 
2671 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2672 {
2673 	struct mlxsw_sp_netevent_work *net_work =
2674 		container_of(work, struct mlxsw_sp_netevent_work, work);
2675 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2676 	struct mlxsw_sp_neigh_entry *neigh_entry;
2677 	struct neighbour *n = net_work->n;
2678 	unsigned char ha[ETH_ALEN];
2679 	bool entry_connected;
2680 	u8 nud_state, dead;
2681 
2682 	/* If these parameters are changed after we release the lock,
2683 	 * then we are guaranteed to receive another event letting us
2684 	 * know about it.
2685 	 */
2686 	read_lock_bh(&n->lock);
2687 	memcpy(ha, n->ha, ETH_ALEN);
2688 	nud_state = n->nud_state;
2689 	dead = n->dead;
2690 	read_unlock_bh(&n->lock);
2691 
2692 	mutex_lock(&mlxsw_sp->router->lock);
2693 	mlxsw_sp_span_respin(mlxsw_sp);
2694 
2695 	entry_connected = nud_state & NUD_VALID && !dead;
2696 	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2697 	if (!entry_connected && !neigh_entry)
2698 		goto out;
2699 	if (!neigh_entry) {
2700 		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2701 		if (IS_ERR(neigh_entry))
2702 			goto out;
2703 	}
2704 
2705 	if (neigh_entry->connected && entry_connected &&
2706 	    !memcmp(neigh_entry->ha, ha, ETH_ALEN))
2707 		goto out;
2708 
2709 	memcpy(neigh_entry->ha, ha, ETH_ALEN);
2710 	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2711 	mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2712 				      dead);
2713 
2714 	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2715 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2716 
2717 out:
2718 	mutex_unlock(&mlxsw_sp->router->lock);
2719 	neigh_release(n);
2720 	kfree(net_work);
2721 }
2722 
2723 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2724 
2725 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2726 {
2727 	struct mlxsw_sp_netevent_work *net_work =
2728 		container_of(work, struct mlxsw_sp_netevent_work, work);
2729 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2730 
2731 	mlxsw_sp_mp_hash_init(mlxsw_sp);
2732 	kfree(net_work);
2733 }
2734 
2735 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2736 
2737 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2738 {
2739 	struct mlxsw_sp_netevent_work *net_work =
2740 		container_of(work, struct mlxsw_sp_netevent_work, work);
2741 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2742 
2743 	__mlxsw_sp_router_init(mlxsw_sp);
2744 	kfree(net_work);
2745 }
2746 
2747 static int mlxsw_sp_router_schedule_work(struct net *net,
2748 					 struct notifier_block *nb,
2749 					 void (*cb)(struct work_struct *))
2750 {
2751 	struct mlxsw_sp_netevent_work *net_work;
2752 	struct mlxsw_sp_router *router;
2753 
2754 	router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2755 	if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2756 		return NOTIFY_DONE;
2757 
2758 	net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2759 	if (!net_work)
2760 		return NOTIFY_BAD;
2761 
2762 	INIT_WORK(&net_work->work, cb);
2763 	net_work->mlxsw_sp = router->mlxsw_sp;
2764 	mlxsw_core_schedule_work(&net_work->work);
2765 	return NOTIFY_DONE;
2766 }
2767 
2768 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2769 					  unsigned long event, void *ptr)
2770 {
2771 	struct mlxsw_sp_netevent_work *net_work;
2772 	struct mlxsw_sp_port *mlxsw_sp_port;
2773 	struct mlxsw_sp *mlxsw_sp;
2774 	unsigned long interval;
2775 	struct neigh_parms *p;
2776 	struct neighbour *n;
2777 
2778 	switch (event) {
2779 	case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2780 		p = ptr;
2781 
2782 		/* We don't care about changes in the default table. */
2783 		if (!p->dev || (p->tbl->family != AF_INET &&
2784 				p->tbl->family != AF_INET6))
2785 			return NOTIFY_DONE;
2786 
2787 		/* We are in atomic context and can't take RTNL mutex,
2788 		 * so use RCU variant to walk the device chain.
2789 		 */
2790 		mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2791 		if (!mlxsw_sp_port)
2792 			return NOTIFY_DONE;
2793 
2794 		mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2795 		interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2796 		mlxsw_sp->router->neighs_update.interval = interval;
2797 
2798 		mlxsw_sp_port_dev_put(mlxsw_sp_port);
2799 		break;
2800 	case NETEVENT_NEIGH_UPDATE:
2801 		n = ptr;
2802 
2803 		if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2804 			return NOTIFY_DONE;
2805 
2806 		mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2807 		if (!mlxsw_sp_port)
2808 			return NOTIFY_DONE;
2809 
2810 		net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2811 		if (!net_work) {
2812 			mlxsw_sp_port_dev_put(mlxsw_sp_port);
2813 			return NOTIFY_BAD;
2814 		}
2815 
2816 		INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2817 		net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2818 		net_work->n = n;
2819 
2820 		/* Take a reference to ensure the neighbour won't be
2821 		 * destructed until we drop the reference in delayed
2822 		 * work.
2823 		 */
2824 		neigh_clone(n);
2825 		mlxsw_core_schedule_work(&net_work->work);
2826 		mlxsw_sp_port_dev_put(mlxsw_sp_port);
2827 		break;
2828 	case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2829 	case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2830 		return mlxsw_sp_router_schedule_work(ptr, nb,
2831 				mlxsw_sp_router_mp_hash_event_work);
2832 
2833 	case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2834 		return mlxsw_sp_router_schedule_work(ptr, nb,
2835 				mlxsw_sp_router_update_priority_work);
2836 	}
2837 
2838 	return NOTIFY_DONE;
2839 }
2840 
2841 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2842 {
2843 	int err;
2844 
2845 	err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2846 			      &mlxsw_sp_neigh_ht_params);
2847 	if (err)
2848 		return err;
2849 
2850 	/* Initialize the polling interval according to the default
2851 	 * table.
2852 	 */
2853 	mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2854 
2855 	/* Create the delayed works for the activity_update */
2856 	INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2857 			  mlxsw_sp_router_neighs_update_work);
2858 	INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2859 			  mlxsw_sp_router_probe_unresolved_nexthops);
2860 	atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0);
2861 	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2862 	mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2863 	return 0;
2864 }
2865 
2866 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2867 {
2868 	cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2869 	cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2870 	rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2871 }
2872 
2873 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2874 					 struct mlxsw_sp_rif *rif)
2875 {
2876 	struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2877 
2878 	list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2879 				 rif_list_node) {
2880 		mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2881 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2882 	}
2883 }
2884 
2885 enum mlxsw_sp_nexthop_type {
2886 	MLXSW_SP_NEXTHOP_TYPE_ETH,
2887 	MLXSW_SP_NEXTHOP_TYPE_IPIP,
2888 };
2889 
2890 enum mlxsw_sp_nexthop_action {
2891 	/* Nexthop forwards packets to an egress RIF */
2892 	MLXSW_SP_NEXTHOP_ACTION_FORWARD,
2893 	/* Nexthop discards packets */
2894 	MLXSW_SP_NEXTHOP_ACTION_DISCARD,
2895 	/* Nexthop traps packets */
2896 	MLXSW_SP_NEXTHOP_ACTION_TRAP,
2897 };
2898 
2899 struct mlxsw_sp_nexthop_key {
2900 	struct fib_nh *fib_nh;
2901 };
2902 
2903 struct mlxsw_sp_nexthop {
2904 	struct list_head neigh_list_node; /* member of neigh entry list */
2905 	struct list_head rif_list_node;
2906 	struct list_head router_list_node;
2907 	struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
2908 						   * this nexthop belongs to
2909 						   */
2910 	struct rhash_head ht_node;
2911 	struct neigh_table *neigh_tbl;
2912 	struct mlxsw_sp_nexthop_key key;
2913 	unsigned char gw_addr[sizeof(struct in6_addr)];
2914 	int ifindex;
2915 	int nh_weight;
2916 	int norm_nh_weight;
2917 	int num_adj_entries;
2918 	struct mlxsw_sp_rif *rif;
2919 	u8 should_offload:1, /* set indicates this nexthop should be written
2920 			      * to the adjacency table.
2921 			      */
2922 	   offloaded:1, /* set indicates this nexthop was written to the
2923 			 * adjacency table.
2924 			 */
2925 	   update:1; /* set indicates this nexthop should be updated in the
2926 		      * adjacency table (f.e., its MAC changed).
2927 		      */
2928 	enum mlxsw_sp_nexthop_action action;
2929 	enum mlxsw_sp_nexthop_type type;
2930 	union {
2931 		struct mlxsw_sp_neigh_entry *neigh_entry;
2932 		struct mlxsw_sp_ipip_entry *ipip_entry;
2933 	};
2934 	unsigned int counter_index;
2935 	bool counter_valid;
2936 };
2937 
2938 enum mlxsw_sp_nexthop_group_type {
2939 	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
2940 	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
2941 	MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
2942 };
2943 
2944 struct mlxsw_sp_nexthop_group_info {
2945 	struct mlxsw_sp_nexthop_group *nh_grp;
2946 	u32 adj_index;
2947 	u16 ecmp_size;
2948 	u16 count;
2949 	int sum_norm_weight;
2950 	u8 adj_index_valid:1,
2951 	   gateway:1, /* routes using the group use a gateway */
2952 	   is_resilient:1;
2953 	struct list_head list; /* member in nh_res_grp_list */
2954 	struct mlxsw_sp_nexthop nexthops[];
2955 #define nh_rif	nexthops[0].rif
2956 };
2957 
2958 struct mlxsw_sp_nexthop_group_vr_key {
2959 	u16 vr_id;
2960 	enum mlxsw_sp_l3proto proto;
2961 };
2962 
2963 struct mlxsw_sp_nexthop_group_vr_entry {
2964 	struct list_head list; /* member in vr_list */
2965 	struct rhash_head ht_node; /* member in vr_ht */
2966 	refcount_t ref_count;
2967 	struct mlxsw_sp_nexthop_group_vr_key key;
2968 };
2969 
2970 struct mlxsw_sp_nexthop_group {
2971 	struct rhash_head ht_node;
2972 	struct list_head fib_list; /* list of fib entries that use this group */
2973 	union {
2974 		struct {
2975 			struct fib_info *fi;
2976 		} ipv4;
2977 		struct {
2978 			u32 id;
2979 		} obj;
2980 	};
2981 	struct mlxsw_sp_nexthop_group_info *nhgi;
2982 	struct list_head vr_list;
2983 	struct rhashtable vr_ht;
2984 	enum mlxsw_sp_nexthop_group_type type;
2985 	bool can_destroy;
2986 };
2987 
2988 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2989 				    struct mlxsw_sp_nexthop *nh)
2990 {
2991 	struct devlink *devlink;
2992 
2993 	devlink = priv_to_devlink(mlxsw_sp->core);
2994 	if (!devlink_dpipe_table_counter_enabled(devlink,
2995 						 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2996 		return;
2997 
2998 	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2999 		return;
3000 
3001 	nh->counter_valid = true;
3002 }
3003 
3004 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
3005 				   struct mlxsw_sp_nexthop *nh)
3006 {
3007 	if (!nh->counter_valid)
3008 		return;
3009 	mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
3010 	nh->counter_valid = false;
3011 }
3012 
3013 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
3014 				 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
3015 {
3016 	if (!nh->counter_valid)
3017 		return -EINVAL;
3018 
3019 	return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
3020 					 p_counter, NULL);
3021 }
3022 
3023 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
3024 					       struct mlxsw_sp_nexthop *nh)
3025 {
3026 	if (!nh) {
3027 		if (list_empty(&router->nexthop_list))
3028 			return NULL;
3029 		else
3030 			return list_first_entry(&router->nexthop_list,
3031 						typeof(*nh), router_list_node);
3032 	}
3033 	if (list_is_last(&nh->router_list_node, &router->nexthop_list))
3034 		return NULL;
3035 	return list_next_entry(nh, router_list_node);
3036 }
3037 
3038 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
3039 {
3040 	return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3041 }
3042 
3043 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3044 {
3045 	if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3046 	    !mlxsw_sp_nexthop_is_forward(nh))
3047 		return NULL;
3048 	return nh->neigh_entry->ha;
3049 }
3050 
3051 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3052 			     u32 *p_adj_size, u32 *p_adj_hash_index)
3053 {
3054 	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3055 	u32 adj_hash_index = 0;
3056 	int i;
3057 
3058 	if (!nh->offloaded || !nhgi->adj_index_valid)
3059 		return -EINVAL;
3060 
3061 	*p_adj_index = nhgi->adj_index;
3062 	*p_adj_size = nhgi->ecmp_size;
3063 
3064 	for (i = 0; i < nhgi->count; i++) {
3065 		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3066 
3067 		if (nh_iter == nh)
3068 			break;
3069 		if (nh_iter->offloaded)
3070 			adj_hash_index += nh_iter->num_adj_entries;
3071 	}
3072 
3073 	*p_adj_hash_index = adj_hash_index;
3074 	return 0;
3075 }
3076 
3077 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3078 {
3079 	return nh->rif;
3080 }
3081 
3082 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3083 {
3084 	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3085 	int i;
3086 
3087 	for (i = 0; i < nhgi->count; i++) {
3088 		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3089 
3090 		if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3091 			return true;
3092 	}
3093 	return false;
3094 }
3095 
3096 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3097 	.key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3098 	.head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3099 	.key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3100 	.automatic_shrinking = true,
3101 };
3102 
3103 static struct mlxsw_sp_nexthop_group_vr_entry *
3104 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3105 				       const struct mlxsw_sp_fib *fib)
3106 {
3107 	struct mlxsw_sp_nexthop_group_vr_key key;
3108 
3109 	memset(&key, 0, sizeof(key));
3110 	key.vr_id = fib->vr->id;
3111 	key.proto = fib->proto;
3112 	return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
3113 				      mlxsw_sp_nexthop_group_vr_ht_params);
3114 }
3115 
3116 static int
3117 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3118 				       const struct mlxsw_sp_fib *fib)
3119 {
3120 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3121 	int err;
3122 
3123 	vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
3124 	if (!vr_entry)
3125 		return -ENOMEM;
3126 
3127 	vr_entry->key.vr_id = fib->vr->id;
3128 	vr_entry->key.proto = fib->proto;
3129 	refcount_set(&vr_entry->ref_count, 1);
3130 
3131 	err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3132 				     mlxsw_sp_nexthop_group_vr_ht_params);
3133 	if (err)
3134 		goto err_hashtable_insert;
3135 
3136 	list_add(&vr_entry->list, &nh_grp->vr_list);
3137 
3138 	return 0;
3139 
3140 err_hashtable_insert:
3141 	kfree(vr_entry);
3142 	return err;
3143 }
3144 
3145 static void
3146 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3147 					struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3148 {
3149 	list_del(&vr_entry->list);
3150 	rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3151 			       mlxsw_sp_nexthop_group_vr_ht_params);
3152 	kfree(vr_entry);
3153 }
3154 
3155 static int
3156 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3157 			       const struct mlxsw_sp_fib *fib)
3158 {
3159 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3160 
3161 	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3162 	if (vr_entry) {
3163 		refcount_inc(&vr_entry->ref_count);
3164 		return 0;
3165 	}
3166 
3167 	return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3168 }
3169 
3170 static void
3171 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3172 				 const struct mlxsw_sp_fib *fib)
3173 {
3174 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3175 
3176 	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3177 	if (WARN_ON_ONCE(!vr_entry))
3178 		return;
3179 
3180 	if (!refcount_dec_and_test(&vr_entry->ref_count))
3181 		return;
3182 
3183 	mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3184 }
3185 
3186 struct mlxsw_sp_nexthop_group_cmp_arg {
3187 	enum mlxsw_sp_nexthop_group_type type;
3188 	union {
3189 		struct fib_info *fi;
3190 		struct mlxsw_sp_fib6_entry *fib6_entry;
3191 		u32 id;
3192 	};
3193 };
3194 
3195 static bool
3196 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3197 				    const struct in6_addr *gw, int ifindex,
3198 				    int weight)
3199 {
3200 	int i;
3201 
3202 	for (i = 0; i < nh_grp->nhgi->count; i++) {
3203 		const struct mlxsw_sp_nexthop *nh;
3204 
3205 		nh = &nh_grp->nhgi->nexthops[i];
3206 		if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3207 		    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3208 			return true;
3209 	}
3210 
3211 	return false;
3212 }
3213 
3214 static bool
3215 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3216 			    const struct mlxsw_sp_fib6_entry *fib6_entry)
3217 {
3218 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3219 
3220 	if (nh_grp->nhgi->count != fib6_entry->nrt6)
3221 		return false;
3222 
3223 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3224 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3225 		struct in6_addr *gw;
3226 		int ifindex, weight;
3227 
3228 		ifindex = fib6_nh->fib_nh_dev->ifindex;
3229 		weight = fib6_nh->fib_nh_weight;
3230 		gw = &fib6_nh->fib_nh_gw6;
3231 		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3232 							 weight))
3233 			return false;
3234 	}
3235 
3236 	return true;
3237 }
3238 
3239 static int
3240 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3241 {
3242 	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3243 	const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3244 
3245 	if (nh_grp->type != cmp_arg->type)
3246 		return 1;
3247 
3248 	switch (cmp_arg->type) {
3249 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3250 		return cmp_arg->fi != nh_grp->ipv4.fi;
3251 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3252 		return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3253 						    cmp_arg->fib6_entry);
3254 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3255 		return cmp_arg->id != nh_grp->obj.id;
3256 	default:
3257 		WARN_ON(1);
3258 		return 1;
3259 	}
3260 }
3261 
3262 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3263 {
3264 	const struct mlxsw_sp_nexthop_group *nh_grp = data;
3265 	const struct mlxsw_sp_nexthop *nh;
3266 	struct fib_info *fi;
3267 	unsigned int val;
3268 	int i;
3269 
3270 	switch (nh_grp->type) {
3271 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3272 		fi = nh_grp->ipv4.fi;
3273 		return jhash(&fi, sizeof(fi), seed);
3274 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3275 		val = nh_grp->nhgi->count;
3276 		for (i = 0; i < nh_grp->nhgi->count; i++) {
3277 			nh = &nh_grp->nhgi->nexthops[i];
3278 			val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3279 			val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3280 		}
3281 		return jhash(&val, sizeof(val), seed);
3282 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3283 		return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
3284 	default:
3285 		WARN_ON(1);
3286 		return 0;
3287 	}
3288 }
3289 
3290 static u32
3291 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3292 {
3293 	unsigned int val = fib6_entry->nrt6;
3294 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3295 
3296 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3297 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3298 		struct net_device *dev = fib6_nh->fib_nh_dev;
3299 		struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3300 
3301 		val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3302 		val ^= jhash(gw, sizeof(*gw), seed);
3303 	}
3304 
3305 	return jhash(&val, sizeof(val), seed);
3306 }
3307 
3308 static u32
3309 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3310 {
3311 	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3312 
3313 	switch (cmp_arg->type) {
3314 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3315 		return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3316 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3317 		return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3318 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3319 		return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
3320 	default:
3321 		WARN_ON(1);
3322 		return 0;
3323 	}
3324 }
3325 
3326 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3327 	.head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3328 	.hashfn	     = mlxsw_sp_nexthop_group_hash,
3329 	.obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3330 	.obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3331 };
3332 
3333 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3334 					 struct mlxsw_sp_nexthop_group *nh_grp)
3335 {
3336 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3337 	    !nh_grp->nhgi->gateway)
3338 		return 0;
3339 
3340 	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3341 				      &nh_grp->ht_node,
3342 				      mlxsw_sp_nexthop_group_ht_params);
3343 }
3344 
3345 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3346 					  struct mlxsw_sp_nexthop_group *nh_grp)
3347 {
3348 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3349 	    !nh_grp->nhgi->gateway)
3350 		return;
3351 
3352 	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3353 			       &nh_grp->ht_node,
3354 			       mlxsw_sp_nexthop_group_ht_params);
3355 }
3356 
3357 static struct mlxsw_sp_nexthop_group *
3358 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3359 			       struct fib_info *fi)
3360 {
3361 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3362 
3363 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3364 	cmp_arg.fi = fi;
3365 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3366 				      &cmp_arg,
3367 				      mlxsw_sp_nexthop_group_ht_params);
3368 }
3369 
3370 static struct mlxsw_sp_nexthop_group *
3371 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3372 			       struct mlxsw_sp_fib6_entry *fib6_entry)
3373 {
3374 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3375 
3376 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3377 	cmp_arg.fib6_entry = fib6_entry;
3378 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3379 				      &cmp_arg,
3380 				      mlxsw_sp_nexthop_group_ht_params);
3381 }
3382 
3383 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3384 	.key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3385 	.head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3386 	.key_len = sizeof(struct mlxsw_sp_nexthop_key),
3387 };
3388 
3389 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3390 				   struct mlxsw_sp_nexthop *nh)
3391 {
3392 	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3393 				      &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3394 }
3395 
3396 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3397 				    struct mlxsw_sp_nexthop *nh)
3398 {
3399 	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3400 			       mlxsw_sp_nexthop_ht_params);
3401 }
3402 
3403 static struct mlxsw_sp_nexthop *
3404 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3405 			struct mlxsw_sp_nexthop_key key)
3406 {
3407 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3408 				      mlxsw_sp_nexthop_ht_params);
3409 }
3410 
3411 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3412 					     enum mlxsw_sp_l3proto proto,
3413 					     u16 vr_id,
3414 					     u32 adj_index, u16 ecmp_size,
3415 					     u32 new_adj_index,
3416 					     u16 new_ecmp_size)
3417 {
3418 	char raleu_pl[MLXSW_REG_RALEU_LEN];
3419 
3420 	mlxsw_reg_raleu_pack(raleu_pl,
3421 			     (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
3422 			     adj_index, ecmp_size, new_adj_index,
3423 			     new_ecmp_size);
3424 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3425 }
3426 
3427 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3428 					  struct mlxsw_sp_nexthop_group *nh_grp,
3429 					  u32 old_adj_index, u16 old_ecmp_size)
3430 {
3431 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3432 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3433 	int err;
3434 
3435 	list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3436 		err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3437 							vr_entry->key.proto,
3438 							vr_entry->key.vr_id,
3439 							old_adj_index,
3440 							old_ecmp_size,
3441 							nhgi->adj_index,
3442 							nhgi->ecmp_size);
3443 		if (err)
3444 			goto err_mass_update_vr;
3445 	}
3446 	return 0;
3447 
3448 err_mass_update_vr:
3449 	list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3450 		mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
3451 						  vr_entry->key.vr_id,
3452 						  nhgi->adj_index,
3453 						  nhgi->ecmp_size,
3454 						  old_adj_index, old_ecmp_size);
3455 	return err;
3456 }
3457 
3458 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3459 					 u32 adj_index,
3460 					 struct mlxsw_sp_nexthop *nh,
3461 					 bool force, char *ratr_pl)
3462 {
3463 	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3464 	enum mlxsw_reg_ratr_op op;
3465 	u16 rif_index;
3466 
3467 	rif_index = nh->rif ? nh->rif->rif_index :
3468 			      mlxsw_sp->router->lb_rif_index;
3469 	op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
3470 		     MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
3471 	mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET,
3472 			    adj_index, rif_index);
3473 	switch (nh->action) {
3474 	case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3475 		mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3476 		break;
3477 	case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3478 		mlxsw_reg_ratr_trap_action_set(ratr_pl,
3479 					       MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3480 		break;
3481 	case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3482 		mlxsw_reg_ratr_trap_action_set(ratr_pl,
3483 					       MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3484 		mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
3485 		break;
3486 	default:
3487 		WARN_ON_ONCE(1);
3488 		return -EINVAL;
3489 	}
3490 	if (nh->counter_valid)
3491 		mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3492 	else
3493 		mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3494 
3495 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3496 }
3497 
3498 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3499 				struct mlxsw_sp_nexthop *nh, bool force,
3500 				char *ratr_pl)
3501 {
3502 	int i;
3503 
3504 	for (i = 0; i < nh->num_adj_entries; i++) {
3505 		int err;
3506 
3507 		err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
3508 						    nh, force, ratr_pl);
3509 		if (err)
3510 			return err;
3511 	}
3512 
3513 	return 0;
3514 }
3515 
3516 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3517 					  u32 adj_index,
3518 					  struct mlxsw_sp_nexthop *nh,
3519 					  bool force, char *ratr_pl)
3520 {
3521 	const struct mlxsw_sp_ipip_ops *ipip_ops;
3522 
3523 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3524 	return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
3525 					force, ratr_pl);
3526 }
3527 
3528 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3529 					u32 adj_index,
3530 					struct mlxsw_sp_nexthop *nh, bool force,
3531 					char *ratr_pl)
3532 {
3533 	int i;
3534 
3535 	for (i = 0; i < nh->num_adj_entries; i++) {
3536 		int err;
3537 
3538 		err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3539 						     nh, force, ratr_pl);
3540 		if (err)
3541 			return err;
3542 	}
3543 
3544 	return 0;
3545 }
3546 
3547 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3548 				   struct mlxsw_sp_nexthop *nh, bool force,
3549 				   char *ratr_pl)
3550 {
3551 	/* When action is discard or trap, the nexthop must be
3552 	 * programmed as an Ethernet nexthop.
3553 	 */
3554 	if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3555 	    nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3556 	    nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3557 		return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
3558 						   force, ratr_pl);
3559 	else
3560 		return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
3561 						    force, ratr_pl);
3562 }
3563 
3564 static int
3565 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3566 			      struct mlxsw_sp_nexthop_group_info *nhgi,
3567 			      bool reallocate)
3568 {
3569 	char ratr_pl[MLXSW_REG_RATR_LEN];
3570 	u32 adj_index = nhgi->adj_index; /* base */
3571 	struct mlxsw_sp_nexthop *nh;
3572 	int i;
3573 
3574 	for (i = 0; i < nhgi->count; i++) {
3575 		nh = &nhgi->nexthops[i];
3576 
3577 		if (!nh->should_offload) {
3578 			nh->offloaded = 0;
3579 			continue;
3580 		}
3581 
3582 		if (nh->update || reallocate) {
3583 			int err = 0;
3584 
3585 			err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
3586 						      true, ratr_pl);
3587 			if (err)
3588 				return err;
3589 			nh->update = 0;
3590 			nh->offloaded = 1;
3591 		}
3592 		adj_index += nh->num_adj_entries;
3593 	}
3594 	return 0;
3595 }
3596 
3597 static int
3598 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3599 				    struct mlxsw_sp_nexthop_group *nh_grp)
3600 {
3601 	struct mlxsw_sp_fib_entry *fib_entry;
3602 	int err;
3603 
3604 	list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3605 		err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3606 		if (err)
3607 			return err;
3608 	}
3609 	return 0;
3610 }
3611 
3612 struct mlxsw_sp_adj_grp_size_range {
3613 	u16 start; /* Inclusive */
3614 	u16 end; /* Inclusive */
3615 };
3616 
3617 /* Ordered by range start value */
3618 static const struct mlxsw_sp_adj_grp_size_range
3619 mlxsw_sp1_adj_grp_size_ranges[] = {
3620 	{ .start = 1, .end = 64 },
3621 	{ .start = 512, .end = 512 },
3622 	{ .start = 1024, .end = 1024 },
3623 	{ .start = 2048, .end = 2048 },
3624 	{ .start = 4096, .end = 4096 },
3625 };
3626 
3627 /* Ordered by range start value */
3628 static const struct mlxsw_sp_adj_grp_size_range
3629 mlxsw_sp2_adj_grp_size_ranges[] = {
3630 	{ .start = 1, .end = 128 },
3631 	{ .start = 256, .end = 256 },
3632 	{ .start = 512, .end = 512 },
3633 	{ .start = 1024, .end = 1024 },
3634 	{ .start = 2048, .end = 2048 },
3635 	{ .start = 4096, .end = 4096 },
3636 };
3637 
3638 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
3639 					   u16 *p_adj_grp_size)
3640 {
3641 	int i;
3642 
3643 	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
3644 		const struct mlxsw_sp_adj_grp_size_range *size_range;
3645 
3646 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3647 
3648 		if (*p_adj_grp_size >= size_range->start &&
3649 		    *p_adj_grp_size <= size_range->end)
3650 			return;
3651 
3652 		if (*p_adj_grp_size <= size_range->end) {
3653 			*p_adj_grp_size = size_range->end;
3654 			return;
3655 		}
3656 	}
3657 }
3658 
3659 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
3660 					     u16 *p_adj_grp_size,
3661 					     unsigned int alloc_size)
3662 {
3663 	int i;
3664 
3665 	for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
3666 		const struct mlxsw_sp_adj_grp_size_range *size_range;
3667 
3668 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3669 
3670 		if (alloc_size >= size_range->end) {
3671 			*p_adj_grp_size = size_range->end;
3672 			return;
3673 		}
3674 	}
3675 }
3676 
3677 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3678 				     u16 *p_adj_grp_size)
3679 {
3680 	unsigned int alloc_size;
3681 	int err;
3682 
3683 	/* Round up the requested group size to the next size supported
3684 	 * by the device and make sure the request can be satisfied.
3685 	 */
3686 	mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
3687 	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3688 					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3689 					      *p_adj_grp_size, &alloc_size);
3690 	if (err)
3691 		return err;
3692 	/* It is possible the allocation results in more allocated
3693 	 * entries than requested. Try to use as much of them as
3694 	 * possible.
3695 	 */
3696 	mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
3697 
3698 	return 0;
3699 }
3700 
3701 static void
3702 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3703 {
3704 	int i, g = 0, sum_norm_weight = 0;
3705 	struct mlxsw_sp_nexthop *nh;
3706 
3707 	for (i = 0; i < nhgi->count; i++) {
3708 		nh = &nhgi->nexthops[i];
3709 
3710 		if (!nh->should_offload)
3711 			continue;
3712 		if (g > 0)
3713 			g = gcd(nh->nh_weight, g);
3714 		else
3715 			g = nh->nh_weight;
3716 	}
3717 
3718 	for (i = 0; i < nhgi->count; i++) {
3719 		nh = &nhgi->nexthops[i];
3720 
3721 		if (!nh->should_offload)
3722 			continue;
3723 		nh->norm_nh_weight = nh->nh_weight / g;
3724 		sum_norm_weight += nh->norm_nh_weight;
3725 	}
3726 
3727 	nhgi->sum_norm_weight = sum_norm_weight;
3728 }
3729 
3730 static void
3731 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3732 {
3733 	int i, weight = 0, lower_bound = 0;
3734 	int total = nhgi->sum_norm_weight;
3735 	u16 ecmp_size = nhgi->ecmp_size;
3736 
3737 	for (i = 0; i < nhgi->count; i++) {
3738 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3739 		int upper_bound;
3740 
3741 		if (!nh->should_offload)
3742 			continue;
3743 		weight += nh->norm_nh_weight;
3744 		upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3745 		nh->num_adj_entries = upper_bound - lower_bound;
3746 		lower_bound = upper_bound;
3747 	}
3748 }
3749 
3750 static struct mlxsw_sp_nexthop *
3751 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3752 		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3753 
3754 static void
3755 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3756 					struct mlxsw_sp_nexthop_group *nh_grp)
3757 {
3758 	int i;
3759 
3760 	for (i = 0; i < nh_grp->nhgi->count; i++) {
3761 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3762 
3763 		if (nh->offloaded)
3764 			nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3765 		else
3766 			nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3767 	}
3768 }
3769 
3770 static void
3771 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3772 					  struct mlxsw_sp_fib6_entry *fib6_entry)
3773 {
3774 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3775 
3776 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3777 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3778 		struct mlxsw_sp_nexthop *nh;
3779 
3780 		nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3781 		if (nh && nh->offloaded)
3782 			fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3783 		else
3784 			fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3785 	}
3786 }
3787 
3788 static void
3789 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3790 					struct mlxsw_sp_nexthop_group *nh_grp)
3791 {
3792 	struct mlxsw_sp_fib6_entry *fib6_entry;
3793 
3794 	/* Unfortunately, in IPv6 the route and the nexthop are described by
3795 	 * the same struct, so we need to iterate over all the routes using the
3796 	 * nexthop group and set / clear the offload indication for them.
3797 	 */
3798 	list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3799 			    common.nexthop_group_node)
3800 		__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3801 }
3802 
3803 static void
3804 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3805 					const struct mlxsw_sp_nexthop *nh,
3806 					u16 bucket_index)
3807 {
3808 	struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
3809 	bool offload = false, trap = false;
3810 
3811 	if (nh->offloaded) {
3812 		if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3813 			trap = true;
3814 		else
3815 			offload = true;
3816 	}
3817 	nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3818 				    bucket_index, offload, trap);
3819 }
3820 
3821 static void
3822 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3823 					   struct mlxsw_sp_nexthop_group *nh_grp)
3824 {
3825 	int i;
3826 
3827 	/* Do not update the flags if the nexthop group is being destroyed
3828 	 * since:
3829 	 * 1. The nexthop objects is being deleted, in which case the flags are
3830 	 * irrelevant.
3831 	 * 2. The nexthop group was replaced by a newer group, in which case
3832 	 * the flags of the nexthop object were already updated based on the
3833 	 * new group.
3834 	 */
3835 	if (nh_grp->can_destroy)
3836 		return;
3837 
3838 	nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3839 			     nh_grp->nhgi->adj_index_valid, false);
3840 
3841 	/* Update flags of individual nexthop buckets in case of a resilient
3842 	 * nexthop group.
3843 	 */
3844 	if (!nh_grp->nhgi->is_resilient)
3845 		return;
3846 
3847 	for (i = 0; i < nh_grp->nhgi->count; i++) {
3848 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3849 
3850 		mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i);
3851 	}
3852 }
3853 
3854 static void
3855 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3856 				       struct mlxsw_sp_nexthop_group *nh_grp)
3857 {
3858 	switch (nh_grp->type) {
3859 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3860 		mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
3861 		break;
3862 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3863 		mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
3864 		break;
3865 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3866 		mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
3867 		break;
3868 	}
3869 }
3870 
3871 static int
3872 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3873 			       struct mlxsw_sp_nexthop_group *nh_grp)
3874 {
3875 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3876 	u16 ecmp_size, old_ecmp_size;
3877 	struct mlxsw_sp_nexthop *nh;
3878 	bool offload_change = false;
3879 	u32 adj_index;
3880 	bool old_adj_index_valid;
3881 	u32 old_adj_index;
3882 	int i, err2, err;
3883 
3884 	if (!nhgi->gateway)
3885 		return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3886 
3887 	for (i = 0; i < nhgi->count; i++) {
3888 		nh = &nhgi->nexthops[i];
3889 
3890 		if (nh->should_offload != nh->offloaded) {
3891 			offload_change = true;
3892 			if (nh->should_offload)
3893 				nh->update = 1;
3894 		}
3895 	}
3896 	if (!offload_change) {
3897 		/* Nothing was added or removed, so no need to reallocate. Just
3898 		 * update MAC on existing adjacency indexes.
3899 		 */
3900 		err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
3901 		if (err) {
3902 			dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3903 			goto set_trap;
3904 		}
3905 		/* Flags of individual nexthop buckets might need to be
3906 		 * updated.
3907 		 */
3908 		mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3909 		return 0;
3910 	}
3911 	mlxsw_sp_nexthop_group_normalize(nhgi);
3912 	if (!nhgi->sum_norm_weight) {
3913 		/* No neigh of this group is connected so we just set
3914 		 * the trap and let everthing flow through kernel.
3915 		 */
3916 		err = 0;
3917 		goto set_trap;
3918 	}
3919 
3920 	ecmp_size = nhgi->sum_norm_weight;
3921 	err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3922 	if (err)
3923 		/* No valid allocation size available. */
3924 		goto set_trap;
3925 
3926 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3927 				  ecmp_size, &adj_index);
3928 	if (err) {
3929 		/* We ran out of KVD linear space, just set the
3930 		 * trap and let everything flow through kernel.
3931 		 */
3932 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3933 		goto set_trap;
3934 	}
3935 	old_adj_index_valid = nhgi->adj_index_valid;
3936 	old_adj_index = nhgi->adj_index;
3937 	old_ecmp_size = nhgi->ecmp_size;
3938 	nhgi->adj_index_valid = 1;
3939 	nhgi->adj_index = adj_index;
3940 	nhgi->ecmp_size = ecmp_size;
3941 	mlxsw_sp_nexthop_group_rebalance(nhgi);
3942 	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
3943 	if (err) {
3944 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3945 		goto set_trap;
3946 	}
3947 
3948 	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3949 
3950 	if (!old_adj_index_valid) {
3951 		/* The trap was set for fib entries, so we have to call
3952 		 * fib entry update to unset it and use adjacency index.
3953 		 */
3954 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3955 		if (err) {
3956 			dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3957 			goto set_trap;
3958 		}
3959 		return 0;
3960 	}
3961 
3962 	err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3963 					     old_adj_index, old_ecmp_size);
3964 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3965 			   old_ecmp_size, old_adj_index);
3966 	if (err) {
3967 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3968 		goto set_trap;
3969 	}
3970 
3971 	return 0;
3972 
3973 set_trap:
3974 	old_adj_index_valid = nhgi->adj_index_valid;
3975 	nhgi->adj_index_valid = 0;
3976 	for (i = 0; i < nhgi->count; i++) {
3977 		nh = &nhgi->nexthops[i];
3978 		nh->offloaded = 0;
3979 	}
3980 	err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3981 	if (err2)
3982 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3983 	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
3984 	if (old_adj_index_valid)
3985 		mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3986 				   nhgi->ecmp_size, nhgi->adj_index);
3987 	return err;
3988 }
3989 
3990 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3991 					    bool removing)
3992 {
3993 	if (!removing) {
3994 		nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3995 		nh->should_offload = 1;
3996 	} else if (nh->nhgi->is_resilient) {
3997 		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
3998 		nh->should_offload = 1;
3999 	} else {
4000 		nh->should_offload = 0;
4001 	}
4002 	nh->update = 1;
4003 }
4004 
4005 static int
4006 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
4007 				    struct mlxsw_sp_neigh_entry *neigh_entry)
4008 {
4009 	struct neighbour *n, *old_n = neigh_entry->key.n;
4010 	struct mlxsw_sp_nexthop *nh;
4011 	bool entry_connected;
4012 	u8 nud_state, dead;
4013 	int err;
4014 
4015 	nh = list_first_entry(&neigh_entry->nexthop_list,
4016 			      struct mlxsw_sp_nexthop, neigh_list_node);
4017 
4018 	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4019 	if (!n) {
4020 		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4021 		if (IS_ERR(n))
4022 			return PTR_ERR(n);
4023 		neigh_event_send(n, NULL);
4024 	}
4025 
4026 	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
4027 	neigh_entry->key.n = n;
4028 	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4029 	if (err)
4030 		goto err_neigh_entry_insert;
4031 
4032 	read_lock_bh(&n->lock);
4033 	nud_state = n->nud_state;
4034 	dead = n->dead;
4035 	read_unlock_bh(&n->lock);
4036 	entry_connected = nud_state & NUD_VALID && !dead;
4037 
4038 	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4039 			    neigh_list_node) {
4040 		neigh_release(old_n);
4041 		neigh_clone(n);
4042 		__mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
4043 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4044 	}
4045 
4046 	neigh_release(n);
4047 
4048 	return 0;
4049 
4050 err_neigh_entry_insert:
4051 	neigh_entry->key.n = old_n;
4052 	mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4053 	neigh_release(n);
4054 	return err;
4055 }
4056 
4057 static void
4058 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
4059 			      struct mlxsw_sp_neigh_entry *neigh_entry,
4060 			      bool removing, bool dead)
4061 {
4062 	struct mlxsw_sp_nexthop *nh;
4063 
4064 	if (list_empty(&neigh_entry->nexthop_list))
4065 		return;
4066 
4067 	if (dead) {
4068 		int err;
4069 
4070 		err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
4071 							  neigh_entry);
4072 		if (err)
4073 			dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
4074 		return;
4075 	}
4076 
4077 	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4078 			    neigh_list_node) {
4079 		__mlxsw_sp_nexthop_neigh_update(nh, removing);
4080 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4081 	}
4082 }
4083 
4084 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
4085 				      struct mlxsw_sp_rif *rif)
4086 {
4087 	if (nh->rif)
4088 		return;
4089 
4090 	nh->rif = rif;
4091 	list_add(&nh->rif_list_node, &rif->nexthop_list);
4092 }
4093 
4094 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
4095 {
4096 	if (!nh->rif)
4097 		return;
4098 
4099 	list_del(&nh->rif_list_node);
4100 	nh->rif = NULL;
4101 }
4102 
4103 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
4104 				       struct mlxsw_sp_nexthop *nh)
4105 {
4106 	struct mlxsw_sp_neigh_entry *neigh_entry;
4107 	struct neighbour *n;
4108 	u8 nud_state, dead;
4109 	int err;
4110 
4111 	if (!nh->nhgi->gateway || nh->neigh_entry)
4112 		return 0;
4113 
4114 	/* Take a reference of neigh here ensuring that neigh would
4115 	 * not be destructed before the nexthop entry is finished.
4116 	 * The reference is taken either in neigh_lookup() or
4117 	 * in neigh_create() in case n is not found.
4118 	 */
4119 	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4120 	if (!n) {
4121 		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, nh->rif->dev);
4122 		if (IS_ERR(n))
4123 			return PTR_ERR(n);
4124 		neigh_event_send(n, NULL);
4125 	}
4126 	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
4127 	if (!neigh_entry) {
4128 		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
4129 		if (IS_ERR(neigh_entry)) {
4130 			err = -EINVAL;
4131 			goto err_neigh_entry_create;
4132 		}
4133 	}
4134 
4135 	/* If that is the first nexthop connected to that neigh, add to
4136 	 * nexthop_neighs_list
4137 	 */
4138 	if (list_empty(&neigh_entry->nexthop_list))
4139 		list_add_tail(&neigh_entry->nexthop_neighs_list_node,
4140 			      &mlxsw_sp->router->nexthop_neighs_list);
4141 
4142 	nh->neigh_entry = neigh_entry;
4143 	list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
4144 	read_lock_bh(&n->lock);
4145 	nud_state = n->nud_state;
4146 	dead = n->dead;
4147 	read_unlock_bh(&n->lock);
4148 	__mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
4149 
4150 	return 0;
4151 
4152 err_neigh_entry_create:
4153 	neigh_release(n);
4154 	return err;
4155 }
4156 
4157 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4158 					struct mlxsw_sp_nexthop *nh)
4159 {
4160 	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4161 	struct neighbour *n;
4162 
4163 	if (!neigh_entry)
4164 		return;
4165 	n = neigh_entry->key.n;
4166 
4167 	__mlxsw_sp_nexthop_neigh_update(nh, true);
4168 	list_del(&nh->neigh_list_node);
4169 	nh->neigh_entry = NULL;
4170 
4171 	/* If that is the last nexthop connected to that neigh, remove from
4172 	 * nexthop_neighs_list
4173 	 */
4174 	if (list_empty(&neigh_entry->nexthop_list))
4175 		list_del(&neigh_entry->nexthop_neighs_list_node);
4176 
4177 	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
4178 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4179 
4180 	neigh_release(n);
4181 }
4182 
4183 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4184 {
4185 	struct net_device *ul_dev;
4186 	bool is_up;
4187 
4188 	rcu_read_lock();
4189 	ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4190 	is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4191 	rcu_read_unlock();
4192 
4193 	return is_up;
4194 }
4195 
4196 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4197 				       struct mlxsw_sp_nexthop *nh,
4198 				       struct mlxsw_sp_ipip_entry *ipip_entry)
4199 {
4200 	bool removing;
4201 
4202 	if (!nh->nhgi->gateway || nh->ipip_entry)
4203 		return;
4204 
4205 	nh->ipip_entry = ipip_entry;
4206 	removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
4207 	__mlxsw_sp_nexthop_neigh_update(nh, removing);
4208 	mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
4209 }
4210 
4211 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4212 				       struct mlxsw_sp_nexthop *nh)
4213 {
4214 	struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4215 
4216 	if (!ipip_entry)
4217 		return;
4218 
4219 	__mlxsw_sp_nexthop_neigh_update(nh, true);
4220 	nh->ipip_entry = NULL;
4221 }
4222 
4223 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4224 					const struct fib_nh *fib_nh,
4225 					enum mlxsw_sp_ipip_type *p_ipipt)
4226 {
4227 	struct net_device *dev = fib_nh->fib_nh_dev;
4228 
4229 	return dev &&
4230 	       fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4231 	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
4232 }
4233 
4234 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4235 				      struct mlxsw_sp_nexthop *nh,
4236 				      const struct net_device *dev)
4237 {
4238 	const struct mlxsw_sp_ipip_ops *ipip_ops;
4239 	struct mlxsw_sp_ipip_entry *ipip_entry;
4240 	struct mlxsw_sp_rif *rif;
4241 	int err;
4242 
4243 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4244 	if (ipip_entry) {
4245 		ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4246 		if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4247 			nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4248 			mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4249 			return 0;
4250 		}
4251 	}
4252 
4253 	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4254 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4255 	if (!rif)
4256 		return 0;
4257 
4258 	mlxsw_sp_nexthop_rif_init(nh, rif);
4259 	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4260 	if (err)
4261 		goto err_neigh_init;
4262 
4263 	return 0;
4264 
4265 err_neigh_init:
4266 	mlxsw_sp_nexthop_rif_fini(nh);
4267 	return err;
4268 }
4269 
4270 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4271 				       struct mlxsw_sp_nexthop *nh)
4272 {
4273 	switch (nh->type) {
4274 	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4275 		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4276 		mlxsw_sp_nexthop_rif_fini(nh);
4277 		break;
4278 	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4279 		mlxsw_sp_nexthop_rif_fini(nh);
4280 		mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4281 		break;
4282 	}
4283 }
4284 
4285 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4286 				  struct mlxsw_sp_nexthop_group *nh_grp,
4287 				  struct mlxsw_sp_nexthop *nh,
4288 				  struct fib_nh *fib_nh)
4289 {
4290 	struct net_device *dev = fib_nh->fib_nh_dev;
4291 	struct in_device *in_dev;
4292 	int err;
4293 
4294 	nh->nhgi = nh_grp->nhgi;
4295 	nh->key.fib_nh = fib_nh;
4296 #ifdef CONFIG_IP_ROUTE_MULTIPATH
4297 	nh->nh_weight = fib_nh->fib_nh_weight;
4298 #else
4299 	nh->nh_weight = 1;
4300 #endif
4301 	memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4302 	nh->neigh_tbl = &arp_tbl;
4303 	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4304 	if (err)
4305 		return err;
4306 
4307 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4308 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4309 
4310 	if (!dev)
4311 		return 0;
4312 	nh->ifindex = dev->ifindex;
4313 
4314 	rcu_read_lock();
4315 	in_dev = __in_dev_get_rcu(dev);
4316 	if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4317 	    fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4318 		rcu_read_unlock();
4319 		return 0;
4320 	}
4321 	rcu_read_unlock();
4322 
4323 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4324 	if (err)
4325 		goto err_nexthop_neigh_init;
4326 
4327 	return 0;
4328 
4329 err_nexthop_neigh_init:
4330 	list_del(&nh->router_list_node);
4331 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4332 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4333 	return err;
4334 }
4335 
4336 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4337 				   struct mlxsw_sp_nexthop *nh)
4338 {
4339 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4340 	list_del(&nh->router_list_node);
4341 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4342 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4343 }
4344 
4345 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4346 				    unsigned long event, struct fib_nh *fib_nh)
4347 {
4348 	struct mlxsw_sp_nexthop_key key;
4349 	struct mlxsw_sp_nexthop *nh;
4350 
4351 	key.fib_nh = fib_nh;
4352 	nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4353 	if (!nh)
4354 		return;
4355 
4356 	switch (event) {
4357 	case FIB_EVENT_NH_ADD:
4358 		mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4359 		break;
4360 	case FIB_EVENT_NH_DEL:
4361 		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4362 		break;
4363 	}
4364 
4365 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4366 }
4367 
4368 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4369 					struct mlxsw_sp_rif *rif)
4370 {
4371 	struct mlxsw_sp_nexthop *nh;
4372 	bool removing;
4373 
4374 	list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
4375 		switch (nh->type) {
4376 		case MLXSW_SP_NEXTHOP_TYPE_ETH:
4377 			removing = false;
4378 			break;
4379 		case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4380 			removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
4381 			break;
4382 		default:
4383 			WARN_ON(1);
4384 			continue;
4385 		}
4386 
4387 		__mlxsw_sp_nexthop_neigh_update(nh, removing);
4388 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4389 	}
4390 }
4391 
4392 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
4393 					 struct mlxsw_sp_rif *old_rif,
4394 					 struct mlxsw_sp_rif *new_rif)
4395 {
4396 	struct mlxsw_sp_nexthop *nh;
4397 
4398 	list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
4399 	list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
4400 		nh->rif = new_rif;
4401 	mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
4402 }
4403 
4404 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4405 					   struct mlxsw_sp_rif *rif)
4406 {
4407 	struct mlxsw_sp_nexthop *nh, *tmp;
4408 
4409 	list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
4410 		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4411 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4412 	}
4413 }
4414 
4415 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp)
4416 {
4417 	enum mlxsw_reg_ratr_trap_action trap_action;
4418 	char ratr_pl[MLXSW_REG_RATR_LEN];
4419 	int err;
4420 
4421 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4422 				  &mlxsw_sp->router->adj_trap_index);
4423 	if (err)
4424 		return err;
4425 
4426 	trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
4427 	mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4428 			    MLXSW_REG_RATR_TYPE_ETHERNET,
4429 			    mlxsw_sp->router->adj_trap_index,
4430 			    mlxsw_sp->router->lb_rif_index);
4431 	mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4432 	mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
4433 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4434 	if (err)
4435 		goto err_ratr_write;
4436 
4437 	return 0;
4438 
4439 err_ratr_write:
4440 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4441 			   mlxsw_sp->router->adj_trap_index);
4442 	return err;
4443 }
4444 
4445 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp)
4446 {
4447 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4448 			   mlxsw_sp->router->adj_trap_index);
4449 }
4450 
4451 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp)
4452 {
4453 	int err;
4454 
4455 	if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups))
4456 		return 0;
4457 
4458 	err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp);
4459 	if (err)
4460 		return err;
4461 
4462 	refcount_set(&mlxsw_sp->router->num_groups, 1);
4463 
4464 	return 0;
4465 }
4466 
4467 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp)
4468 {
4469 	if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups))
4470 		return;
4471 
4472 	mlxsw_sp_adj_trap_entry_fini(mlxsw_sp);
4473 }
4474 
4475 static void
4476 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
4477 			     const struct mlxsw_sp_nexthop_group *nh_grp,
4478 			     unsigned long *activity)
4479 {
4480 	char *ratrad_pl;
4481 	int i, err;
4482 
4483 	ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
4484 	if (!ratrad_pl)
4485 		return;
4486 
4487 	mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index,
4488 			      nh_grp->nhgi->count);
4489 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl);
4490 	if (err)
4491 		goto out;
4492 
4493 	for (i = 0; i < nh_grp->nhgi->count; i++) {
4494 		if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i))
4495 			continue;
4496 		bitmap_set(activity, i, 1);
4497 	}
4498 
4499 out:
4500 	kfree(ratrad_pl);
4501 }
4502 
4503 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
4504 
4505 static void
4506 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
4507 				const struct mlxsw_sp_nexthop_group *nh_grp)
4508 {
4509 	unsigned long *activity;
4510 
4511 	activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL);
4512 	if (!activity)
4513 		return;
4514 
4515 	mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
4516 	nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4517 					nh_grp->nhgi->count, activity);
4518 
4519 	bitmap_free(activity);
4520 }
4521 
4522 static void
4523 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
4524 {
4525 	unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
4526 
4527 	mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw,
4528 			       msecs_to_jiffies(interval));
4529 }
4530 
4531 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
4532 {
4533 	struct mlxsw_sp_nexthop_group_info *nhgi;
4534 	struct mlxsw_sp_router *router;
4535 	bool reschedule = false;
4536 
4537 	router = container_of(work, struct mlxsw_sp_router,
4538 			      nh_grp_activity_dw.work);
4539 
4540 	mutex_lock(&router->lock);
4541 
4542 	list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
4543 		mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp);
4544 		reschedule = true;
4545 	}
4546 
4547 	mutex_unlock(&router->lock);
4548 
4549 	if (!reschedule)
4550 		return;
4551 	mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp);
4552 }
4553 
4554 static int
4555 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4556 				     const struct nh_notifier_single_info *nh,
4557 				     struct netlink_ext_ack *extack)
4558 {
4559 	int err = -EINVAL;
4560 
4561 	if (nh->is_fdb)
4562 		NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4563 	else if (nh->has_encap)
4564 		NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4565 	else
4566 		err = 0;
4567 
4568 	return err;
4569 }
4570 
4571 static int
4572 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4573 					  const struct nh_notifier_single_info *nh,
4574 					  struct netlink_ext_ack *extack)
4575 {
4576 	int err;
4577 
4578 	err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4579 	if (err)
4580 		return err;
4581 
4582 	/* Device only nexthops with an IPIP device are programmed as
4583 	 * encapsulating adjacency entries.
4584 	 */
4585 	if (!nh->gw_family && !nh->is_reject &&
4586 	    !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
4587 		NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4588 		return -EINVAL;
4589 	}
4590 
4591 	return 0;
4592 }
4593 
4594 static int
4595 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4596 				    const struct nh_notifier_grp_info *nh_grp,
4597 				    struct netlink_ext_ack *extack)
4598 {
4599 	int i;
4600 
4601 	if (nh_grp->is_fdb) {
4602 		NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4603 		return -EINVAL;
4604 	}
4605 
4606 	for (i = 0; i < nh_grp->num_nh; i++) {
4607 		const struct nh_notifier_single_info *nh;
4608 		int err;
4609 
4610 		nh = &nh_grp->nh_entries[i].nh;
4611 		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4612 								extack);
4613 		if (err)
4614 			return err;
4615 	}
4616 
4617 	return 0;
4618 }
4619 
4620 static int
4621 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
4622 					     const struct nh_notifier_res_table_info *nh_res_table,
4623 					     struct netlink_ext_ack *extack)
4624 {
4625 	unsigned int alloc_size;
4626 	bool valid_size = false;
4627 	int err, i;
4628 
4629 	if (nh_res_table->num_nh_buckets < 32) {
4630 		NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
4631 		return -EINVAL;
4632 	}
4633 
4634 	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
4635 		const struct mlxsw_sp_adj_grp_size_range *size_range;
4636 
4637 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
4638 
4639 		if (nh_res_table->num_nh_buckets >= size_range->start &&
4640 		    nh_res_table->num_nh_buckets <= size_range->end) {
4641 			valid_size = true;
4642 			break;
4643 		}
4644 	}
4645 
4646 	if (!valid_size) {
4647 		NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
4648 		return -EINVAL;
4649 	}
4650 
4651 	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
4652 					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4653 					      nh_res_table->num_nh_buckets,
4654 					      &alloc_size);
4655 	if (err || nh_res_table->num_nh_buckets != alloc_size) {
4656 		NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
4657 		return -EINVAL;
4658 	}
4659 
4660 	return 0;
4661 }
4662 
4663 static int
4664 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
4665 					const struct nh_notifier_res_table_info *nh_res_table,
4666 					struct netlink_ext_ack *extack)
4667 {
4668 	int err;
4669 	u16 i;
4670 
4671 	err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
4672 							   nh_res_table,
4673 							   extack);
4674 	if (err)
4675 		return err;
4676 
4677 	for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
4678 		const struct nh_notifier_single_info *nh;
4679 		int err;
4680 
4681 		nh = &nh_res_table->nhs[i];
4682 		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4683 								extack);
4684 		if (err)
4685 			return err;
4686 	}
4687 
4688 	return 0;
4689 }
4690 
4691 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4692 					 unsigned long event,
4693 					 struct nh_notifier_info *info)
4694 {
4695 	struct nh_notifier_single_info *nh;
4696 
4697 	if (event != NEXTHOP_EVENT_REPLACE &&
4698 	    event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
4699 	    event != NEXTHOP_EVENT_BUCKET_REPLACE)
4700 		return 0;
4701 
4702 	switch (info->type) {
4703 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4704 		return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
4705 							    info->extack);
4706 	case NH_NOTIFIER_INFO_TYPE_GRP:
4707 		return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4708 							   info->nh_grp,
4709 							   info->extack);
4710 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4711 		return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
4712 							       info->nh_res_table,
4713 							       info->extack);
4714 	case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
4715 		nh = &info->nh_res_bucket->new_nh;
4716 		return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4717 								 info->extack);
4718 	default:
4719 		NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4720 		return -EOPNOTSUPP;
4721 	}
4722 }
4723 
4724 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4725 					    const struct nh_notifier_info *info)
4726 {
4727 	const struct net_device *dev;
4728 
4729 	switch (info->type) {
4730 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4731 		dev = info->nh->dev;
4732 		return info->nh->gw_family || info->nh->is_reject ||
4733 		       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
4734 	case NH_NOTIFIER_INFO_TYPE_GRP:
4735 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4736 		/* Already validated earlier. */
4737 		return true;
4738 	default:
4739 		return false;
4740 	}
4741 }
4742 
4743 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
4744 						struct mlxsw_sp_nexthop *nh)
4745 {
4746 	u16 lb_rif_index = mlxsw_sp->router->lb_rif_index;
4747 
4748 	nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
4749 	nh->should_offload = 1;
4750 	/* While nexthops that discard packets do not forward packets
4751 	 * via an egress RIF, they still need to be programmed using a
4752 	 * valid RIF, so use the loopback RIF created during init.
4753 	 */
4754 	nh->rif = mlxsw_sp->router->rifs[lb_rif_index];
4755 }
4756 
4757 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
4758 						struct mlxsw_sp_nexthop *nh)
4759 {
4760 	nh->rif = NULL;
4761 	nh->should_offload = 0;
4762 }
4763 
4764 static int
4765 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
4766 			  struct mlxsw_sp_nexthop_group *nh_grp,
4767 			  struct mlxsw_sp_nexthop *nh,
4768 			  struct nh_notifier_single_info *nh_obj, int weight)
4769 {
4770 	struct net_device *dev = nh_obj->dev;
4771 	int err;
4772 
4773 	nh->nhgi = nh_grp->nhgi;
4774 	nh->nh_weight = weight;
4775 
4776 	switch (nh_obj->gw_family) {
4777 	case AF_INET:
4778 		memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
4779 		nh->neigh_tbl = &arp_tbl;
4780 		break;
4781 	case AF_INET6:
4782 		memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
4783 #if IS_ENABLED(CONFIG_IPV6)
4784 		nh->neigh_tbl = &nd_tbl;
4785 #endif
4786 		break;
4787 	}
4788 
4789 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4790 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4791 	nh->ifindex = dev->ifindex;
4792 
4793 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4794 	if (err)
4795 		goto err_type_init;
4796 
4797 	if (nh_obj->is_reject)
4798 		mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
4799 
4800 	/* In a resilient nexthop group, all the nexthops must be written to
4801 	 * the adjacency table. Even if they do not have a valid neighbour or
4802 	 * RIF.
4803 	 */
4804 	if (nh_grp->nhgi->is_resilient && !nh->should_offload) {
4805 		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
4806 		nh->should_offload = 1;
4807 	}
4808 
4809 	return 0;
4810 
4811 err_type_init:
4812 	list_del(&nh->router_list_node);
4813 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4814 	return err;
4815 }
4816 
4817 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
4818 				      struct mlxsw_sp_nexthop *nh)
4819 {
4820 	if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
4821 		mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
4822 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4823 	list_del(&nh->router_list_node);
4824 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4825 	nh->should_offload = 0;
4826 }
4827 
4828 static int
4829 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
4830 				     struct mlxsw_sp_nexthop_group *nh_grp,
4831 				     struct nh_notifier_info *info)
4832 {
4833 	struct mlxsw_sp_nexthop_group_info *nhgi;
4834 	struct mlxsw_sp_nexthop *nh;
4835 	bool is_resilient = false;
4836 	unsigned int nhs;
4837 	int err, i;
4838 
4839 	switch (info->type) {
4840 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4841 		nhs = 1;
4842 		break;
4843 	case NH_NOTIFIER_INFO_TYPE_GRP:
4844 		nhs = info->nh_grp->num_nh;
4845 		break;
4846 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4847 		nhs = info->nh_res_table->num_nh_buckets;
4848 		is_resilient = true;
4849 		break;
4850 	default:
4851 		return -EINVAL;
4852 	}
4853 
4854 	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
4855 	if (!nhgi)
4856 		return -ENOMEM;
4857 	nh_grp->nhgi = nhgi;
4858 	nhgi->nh_grp = nh_grp;
4859 	nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
4860 	nhgi->is_resilient = is_resilient;
4861 	nhgi->count = nhs;
4862 	for (i = 0; i < nhgi->count; i++) {
4863 		struct nh_notifier_single_info *nh_obj;
4864 		int weight;
4865 
4866 		nh = &nhgi->nexthops[i];
4867 		switch (info->type) {
4868 		case NH_NOTIFIER_INFO_TYPE_SINGLE:
4869 			nh_obj = info->nh;
4870 			weight = 1;
4871 			break;
4872 		case NH_NOTIFIER_INFO_TYPE_GRP:
4873 			nh_obj = &info->nh_grp->nh_entries[i].nh;
4874 			weight = info->nh_grp->nh_entries[i].weight;
4875 			break;
4876 		case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4877 			nh_obj = &info->nh_res_table->nhs[i];
4878 			weight = 1;
4879 			break;
4880 		default:
4881 			err = -EINVAL;
4882 			goto err_nexthop_obj_init;
4883 		}
4884 		err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
4885 						weight);
4886 		if (err)
4887 			goto err_nexthop_obj_init;
4888 	}
4889 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
4890 	if (err)
4891 		goto err_group_inc;
4892 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4893 	if (err) {
4894 		NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
4895 		goto err_group_refresh;
4896 	}
4897 
4898 	/* Add resilient nexthop groups to a list so that the activity of their
4899 	 * nexthop buckets will be periodically queried and cleared.
4900 	 */
4901 	if (nhgi->is_resilient) {
4902 		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
4903 			mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp);
4904 		list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list);
4905 	}
4906 
4907 	return 0;
4908 
4909 err_group_refresh:
4910 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
4911 err_group_inc:
4912 	i = nhgi->count;
4913 err_nexthop_obj_init:
4914 	for (i--; i >= 0; i--) {
4915 		nh = &nhgi->nexthops[i];
4916 		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
4917 	}
4918 	kfree(nhgi);
4919 	return err;
4920 }
4921 
4922 static void
4923 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
4924 				     struct mlxsw_sp_nexthop_group *nh_grp)
4925 {
4926 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4927 	struct mlxsw_sp_router *router = mlxsw_sp->router;
4928 	int i;
4929 
4930 	if (nhgi->is_resilient) {
4931 		list_del(&nhgi->list);
4932 		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
4933 			cancel_delayed_work(&router->nh_grp_activity_dw);
4934 	}
4935 
4936 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
4937 	for (i = nhgi->count - 1; i >= 0; i--) {
4938 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
4939 
4940 		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
4941 	}
4942 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4943 	WARN_ON_ONCE(nhgi->adj_index_valid);
4944 	kfree(nhgi);
4945 }
4946 
4947 static struct mlxsw_sp_nexthop_group *
4948 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
4949 				  struct nh_notifier_info *info)
4950 {
4951 	struct mlxsw_sp_nexthop_group *nh_grp;
4952 	int err;
4953 
4954 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
4955 	if (!nh_grp)
4956 		return ERR_PTR(-ENOMEM);
4957 	INIT_LIST_HEAD(&nh_grp->vr_list);
4958 	err = rhashtable_init(&nh_grp->vr_ht,
4959 			      &mlxsw_sp_nexthop_group_vr_ht_params);
4960 	if (err)
4961 		goto err_nexthop_group_vr_ht_init;
4962 	INIT_LIST_HEAD(&nh_grp->fib_list);
4963 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
4964 	nh_grp->obj.id = info->id;
4965 
4966 	err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
4967 	if (err)
4968 		goto err_nexthop_group_info_init;
4969 
4970 	nh_grp->can_destroy = false;
4971 
4972 	return nh_grp;
4973 
4974 err_nexthop_group_info_init:
4975 	rhashtable_destroy(&nh_grp->vr_ht);
4976 err_nexthop_group_vr_ht_init:
4977 	kfree(nh_grp);
4978 	return ERR_PTR(err);
4979 }
4980 
4981 static void
4982 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
4983 				   struct mlxsw_sp_nexthop_group *nh_grp)
4984 {
4985 	if (!nh_grp->can_destroy)
4986 		return;
4987 	mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
4988 	WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
4989 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
4990 	rhashtable_destroy(&nh_grp->vr_ht);
4991 	kfree(nh_grp);
4992 }
4993 
4994 static struct mlxsw_sp_nexthop_group *
4995 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
4996 {
4997 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
4998 
4999 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5000 	cmp_arg.id = id;
5001 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
5002 				      &cmp_arg,
5003 				      mlxsw_sp_nexthop_group_ht_params);
5004 }
5005 
5006 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
5007 					  struct mlxsw_sp_nexthop_group *nh_grp)
5008 {
5009 	return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5010 }
5011 
5012 static int
5013 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
5014 				   struct mlxsw_sp_nexthop_group *nh_grp,
5015 				   struct mlxsw_sp_nexthop_group *old_nh_grp,
5016 				   struct netlink_ext_ack *extack)
5017 {
5018 	struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
5019 	struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
5020 	int err;
5021 
5022 	old_nh_grp->nhgi = new_nhgi;
5023 	new_nhgi->nh_grp = old_nh_grp;
5024 	nh_grp->nhgi = old_nhgi;
5025 	old_nhgi->nh_grp = nh_grp;
5026 
5027 	if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5028 		/* Both the old adjacency index and the new one are valid.
5029 		 * Routes are currently using the old one. Tell the device to
5030 		 * replace the old adjacency index with the new one.
5031 		 */
5032 		err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp,
5033 						     old_nhgi->adj_index,
5034 						     old_nhgi->ecmp_size);
5035 		if (err) {
5036 			NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
5037 			goto err_out;
5038 		}
5039 	} else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
5040 		/* The old adjacency index is valid, while the new one is not.
5041 		 * Iterate over all the routes using the group and change them
5042 		 * to trap packets to the CPU.
5043 		 */
5044 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5045 		if (err) {
5046 			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
5047 			goto err_out;
5048 		}
5049 	} else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5050 		/* The old adjacency index is invalid, while the new one is.
5051 		 * Iterate over all the routes using the group and change them
5052 		 * to forward packets using the new valid index.
5053 		 */
5054 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5055 		if (err) {
5056 			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
5057 			goto err_out;
5058 		}
5059 	}
5060 
5061 	/* Make sure the flags are set / cleared based on the new nexthop group
5062 	 * information.
5063 	 */
5064 	mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp);
5065 
5066 	/* At this point 'nh_grp' is just a shell that is not used by anyone
5067 	 * and its nexthop group info is the old info that was just replaced
5068 	 * with the new one. Remove it.
5069 	 */
5070 	nh_grp->can_destroy = true;
5071 	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5072 
5073 	return 0;
5074 
5075 err_out:
5076 	old_nhgi->nh_grp = old_nh_grp;
5077 	nh_grp->nhgi = new_nhgi;
5078 	new_nhgi->nh_grp = nh_grp;
5079 	old_nh_grp->nhgi = old_nhgi;
5080 	return err;
5081 }
5082 
5083 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
5084 				    struct nh_notifier_info *info)
5085 {
5086 	struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
5087 	struct netlink_ext_ack *extack = info->extack;
5088 	int err;
5089 
5090 	nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
5091 	if (IS_ERR(nh_grp))
5092 		return PTR_ERR(nh_grp);
5093 
5094 	old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5095 	if (!old_nh_grp)
5096 		err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
5097 	else
5098 		err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
5099 							 old_nh_grp, extack);
5100 
5101 	if (err) {
5102 		nh_grp->can_destroy = true;
5103 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5104 	}
5105 
5106 	return err;
5107 }
5108 
5109 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
5110 				     struct nh_notifier_info *info)
5111 {
5112 	struct mlxsw_sp_nexthop_group *nh_grp;
5113 
5114 	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5115 	if (!nh_grp)
5116 		return;
5117 
5118 	nh_grp->can_destroy = true;
5119 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5120 
5121 	/* If the group still has routes using it, then defer the delete
5122 	 * operation until the last route using it is deleted.
5123 	 */
5124 	if (!list_empty(&nh_grp->fib_list))
5125 		return;
5126 	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5127 }
5128 
5129 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp,
5130 					     u32 adj_index, char *ratr_pl)
5131 {
5132 	MLXSW_REG_ZERO(ratr, ratr_pl);
5133 	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5134 	mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index);
5135 	mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16);
5136 
5137 	return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
5138 }
5139 
5140 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new)
5141 {
5142 	/* Clear the opcode and activity on both the old and new payload as
5143 	 * they are irrelevant for the comparison.
5144 	 */
5145 	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5146 	mlxsw_reg_ratr_a_set(ratr_pl, 0);
5147 	mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ);
5148 	mlxsw_reg_ratr_a_set(ratr_pl_new, 0);
5149 
5150 	/* If the contents of the adjacency entry are consistent with the
5151 	 * replacement request, then replacement was successful.
5152 	 */
5153 	if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN))
5154 		return 0;
5155 
5156 	return -EINVAL;
5157 }
5158 
5159 static int
5160 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp,
5161 				       struct mlxsw_sp_nexthop *nh,
5162 				       struct nh_notifier_info *info)
5163 {
5164 	u16 bucket_index = info->nh_res_bucket->bucket_index;
5165 	struct netlink_ext_ack *extack = info->extack;
5166 	bool force = info->nh_res_bucket->force;
5167 	char ratr_pl_new[MLXSW_REG_RATR_LEN];
5168 	char ratr_pl[MLXSW_REG_RATR_LEN];
5169 	u32 adj_index;
5170 	int err;
5171 
5172 	/* No point in trying an atomic replacement if the idle timer interval
5173 	 * is smaller than the interval in which we query and clear activity.
5174 	 */
5175 	if (!force && info->nh_res_bucket->idle_timer_ms <
5176 	    MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL)
5177 		force = true;
5178 
5179 	adj_index = nh->nhgi->adj_index + bucket_index;
5180 	err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl);
5181 	if (err) {
5182 		NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket");
5183 		return err;
5184 	}
5185 
5186 	if (!force) {
5187 		err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index,
5188 							ratr_pl_new);
5189 		if (err) {
5190 			NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent");
5191 			return err;
5192 		}
5193 
5194 		err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new);
5195 		if (err) {
5196 			NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement");
5197 			return err;
5198 		}
5199 	}
5200 
5201 	nh->update = 0;
5202 	nh->offloaded = 1;
5203 	mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index);
5204 
5205 	return 0;
5206 }
5207 
5208 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp,
5209 					       struct nh_notifier_info *info)
5210 {
5211 	u16 bucket_index = info->nh_res_bucket->bucket_index;
5212 	struct netlink_ext_ack *extack = info->extack;
5213 	struct mlxsw_sp_nexthop_group_info *nhgi;
5214 	struct nh_notifier_single_info *nh_obj;
5215 	struct mlxsw_sp_nexthop_group *nh_grp;
5216 	struct mlxsw_sp_nexthop *nh;
5217 	int err;
5218 
5219 	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5220 	if (!nh_grp) {
5221 		NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found");
5222 		return -EINVAL;
5223 	}
5224 
5225 	nhgi = nh_grp->nhgi;
5226 
5227 	if (bucket_index >= nhgi->count) {
5228 		NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range");
5229 		return -EINVAL;
5230 	}
5231 
5232 	nh = &nhgi->nexthops[bucket_index];
5233 	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5234 
5235 	nh_obj = &info->nh_res_bucket->new_nh;
5236 	err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5237 	if (err) {
5238 		NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement");
5239 		goto err_nexthop_obj_init;
5240 	}
5241 
5242 	err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info);
5243 	if (err)
5244 		goto err_nexthop_obj_bucket_adj_update;
5245 
5246 	return 0;
5247 
5248 err_nexthop_obj_bucket_adj_update:
5249 	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5250 err_nexthop_obj_init:
5251 	nh_obj = &info->nh_res_bucket->old_nh;
5252 	mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5253 	/* The old adjacency entry was not overwritten */
5254 	nh->update = 0;
5255 	nh->offloaded = 1;
5256 	return err;
5257 }
5258 
5259 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
5260 				      unsigned long event, void *ptr)
5261 {
5262 	struct nh_notifier_info *info = ptr;
5263 	struct mlxsw_sp_router *router;
5264 	int err = 0;
5265 
5266 	router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
5267 	err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info);
5268 	if (err)
5269 		goto out;
5270 
5271 	mutex_lock(&router->lock);
5272 
5273 	switch (event) {
5274 	case NEXTHOP_EVENT_REPLACE:
5275 		err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info);
5276 		break;
5277 	case NEXTHOP_EVENT_DEL:
5278 		mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info);
5279 		break;
5280 	case NEXTHOP_EVENT_BUCKET_REPLACE:
5281 		err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp,
5282 							  info);
5283 		break;
5284 	default:
5285 		break;
5286 	}
5287 
5288 	mutex_unlock(&router->lock);
5289 
5290 out:
5291 	return notifier_from_errno(err);
5292 }
5293 
5294 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5295 				   struct fib_info *fi)
5296 {
5297 	const struct fib_nh *nh = fib_info_nh(fi, 0);
5298 
5299 	return nh->fib_nh_gw_family ||
5300 	       mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
5301 }
5302 
5303 static int
5304 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
5305 				  struct mlxsw_sp_nexthop_group *nh_grp)
5306 {
5307 	unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
5308 	struct mlxsw_sp_nexthop_group_info *nhgi;
5309 	struct mlxsw_sp_nexthop *nh;
5310 	int err, i;
5311 
5312 	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5313 	if (!nhgi)
5314 		return -ENOMEM;
5315 	nh_grp->nhgi = nhgi;
5316 	nhgi->nh_grp = nh_grp;
5317 	nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
5318 	nhgi->count = nhs;
5319 	for (i = 0; i < nhgi->count; i++) {
5320 		struct fib_nh *fib_nh;
5321 
5322 		nh = &nhgi->nexthops[i];
5323 		fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
5324 		err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
5325 		if (err)
5326 			goto err_nexthop4_init;
5327 	}
5328 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5329 	if (err)
5330 		goto err_group_inc;
5331 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5332 	if (err)
5333 		goto err_group_refresh;
5334 
5335 	return 0;
5336 
5337 err_group_refresh:
5338 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5339 err_group_inc:
5340 	i = nhgi->count;
5341 err_nexthop4_init:
5342 	for (i--; i >= 0; i--) {
5343 		nh = &nhgi->nexthops[i];
5344 		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5345 	}
5346 	kfree(nhgi);
5347 	return err;
5348 }
5349 
5350 static void
5351 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5352 				  struct mlxsw_sp_nexthop_group *nh_grp)
5353 {
5354 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5355 	int i;
5356 
5357 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5358 	for (i = nhgi->count - 1; i >= 0; i--) {
5359 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5360 
5361 		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5362 	}
5363 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5364 	WARN_ON_ONCE(nhgi->adj_index_valid);
5365 	kfree(nhgi);
5366 }
5367 
5368 static struct mlxsw_sp_nexthop_group *
5369 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
5370 {
5371 	struct mlxsw_sp_nexthop_group *nh_grp;
5372 	int err;
5373 
5374 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5375 	if (!nh_grp)
5376 		return ERR_PTR(-ENOMEM);
5377 	INIT_LIST_HEAD(&nh_grp->vr_list);
5378 	err = rhashtable_init(&nh_grp->vr_ht,
5379 			      &mlxsw_sp_nexthop_group_vr_ht_params);
5380 	if (err)
5381 		goto err_nexthop_group_vr_ht_init;
5382 	INIT_LIST_HEAD(&nh_grp->fib_list);
5383 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
5384 	nh_grp->ipv4.fi = fi;
5385 	fib_info_hold(fi);
5386 
5387 	err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
5388 	if (err)
5389 		goto err_nexthop_group_info_init;
5390 
5391 	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5392 	if (err)
5393 		goto err_nexthop_group_insert;
5394 
5395 	nh_grp->can_destroy = true;
5396 
5397 	return nh_grp;
5398 
5399 err_nexthop_group_insert:
5400 	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5401 err_nexthop_group_info_init:
5402 	fib_info_put(fi);
5403 	rhashtable_destroy(&nh_grp->vr_ht);
5404 err_nexthop_group_vr_ht_init:
5405 	kfree(nh_grp);
5406 	return ERR_PTR(err);
5407 }
5408 
5409 static void
5410 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
5411 				struct mlxsw_sp_nexthop_group *nh_grp)
5412 {
5413 	if (!nh_grp->can_destroy)
5414 		return;
5415 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5416 	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5417 	fib_info_put(nh_grp->ipv4.fi);
5418 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5419 	rhashtable_destroy(&nh_grp->vr_ht);
5420 	kfree(nh_grp);
5421 }
5422 
5423 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
5424 				       struct mlxsw_sp_fib_entry *fib_entry,
5425 				       struct fib_info *fi)
5426 {
5427 	struct mlxsw_sp_nexthop_group *nh_grp;
5428 
5429 	if (fi->nh) {
5430 		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
5431 							   fi->nh->id);
5432 		if (WARN_ON_ONCE(!nh_grp))
5433 			return -EINVAL;
5434 		goto out;
5435 	}
5436 
5437 	nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
5438 	if (!nh_grp) {
5439 		nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
5440 		if (IS_ERR(nh_grp))
5441 			return PTR_ERR(nh_grp);
5442 	}
5443 out:
5444 	list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
5445 	fib_entry->nh_group = nh_grp;
5446 	return 0;
5447 }
5448 
5449 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
5450 					struct mlxsw_sp_fib_entry *fib_entry)
5451 {
5452 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5453 
5454 	list_del(&fib_entry->nexthop_group_node);
5455 	if (!list_empty(&nh_grp->fib_list))
5456 		return;
5457 
5458 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
5459 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5460 		return;
5461 	}
5462 
5463 	mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
5464 }
5465 
5466 static bool
5467 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5468 {
5469 	struct mlxsw_sp_fib4_entry *fib4_entry;
5470 
5471 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5472 				  common);
5473 	return !fib4_entry->dscp;
5474 }
5475 
5476 static bool
5477 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5478 {
5479 	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5480 
5481 	switch (fib_entry->fib_node->fib->proto) {
5482 	case MLXSW_SP_L3_PROTO_IPV4:
5483 		if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
5484 			return false;
5485 		break;
5486 	case MLXSW_SP_L3_PROTO_IPV6:
5487 		break;
5488 	}
5489 
5490 	switch (fib_entry->type) {
5491 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5492 		return !!nh_group->nhgi->adj_index_valid;
5493 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5494 		return !!nh_group->nhgi->nh_rif;
5495 	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5496 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5497 	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5498 		return true;
5499 	default:
5500 		return false;
5501 	}
5502 }
5503 
5504 static struct mlxsw_sp_nexthop *
5505 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
5506 		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5507 {
5508 	int i;
5509 
5510 	for (i = 0; i < nh_grp->nhgi->count; i++) {
5511 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
5512 		struct fib6_info *rt = mlxsw_sp_rt6->rt;
5513 
5514 		if (nh->rif && nh->rif->dev == rt->fib6_nh->fib_nh_dev &&
5515 		    ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
5516 				    &rt->fib6_nh->fib_nh_gw6))
5517 			return nh;
5518 	}
5519 
5520 	return NULL;
5521 }
5522 
5523 static void
5524 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5525 				      struct fib_entry_notifier_info *fen_info)
5526 {
5527 	u32 *p_dst = (u32 *) &fen_info->dst;
5528 	struct fib_rt_info fri;
5529 
5530 	fri.fi = fen_info->fi;
5531 	fri.tb_id = fen_info->tb_id;
5532 	fri.dst = cpu_to_be32(*p_dst);
5533 	fri.dst_len = fen_info->dst_len;
5534 	fri.dscp = fen_info->dscp;
5535 	fri.type = fen_info->type;
5536 	fri.offload = false;
5537 	fri.trap = false;
5538 	fri.offload_failed = true;
5539 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5540 }
5541 
5542 static void
5543 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5544 				 struct mlxsw_sp_fib_entry *fib_entry)
5545 {
5546 	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5547 	int dst_len = fib_entry->fib_node->key.prefix_len;
5548 	struct mlxsw_sp_fib4_entry *fib4_entry;
5549 	struct fib_rt_info fri;
5550 	bool should_offload;
5551 
5552 	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5553 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5554 				  common);
5555 	fri.fi = fib4_entry->fi;
5556 	fri.tb_id = fib4_entry->tb_id;
5557 	fri.dst = cpu_to_be32(*p_dst);
5558 	fri.dst_len = dst_len;
5559 	fri.dscp = fib4_entry->dscp;
5560 	fri.type = fib4_entry->type;
5561 	fri.offload = should_offload;
5562 	fri.trap = !should_offload;
5563 	fri.offload_failed = false;
5564 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5565 }
5566 
5567 static void
5568 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5569 				   struct mlxsw_sp_fib_entry *fib_entry)
5570 {
5571 	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5572 	int dst_len = fib_entry->fib_node->key.prefix_len;
5573 	struct mlxsw_sp_fib4_entry *fib4_entry;
5574 	struct fib_rt_info fri;
5575 
5576 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5577 				  common);
5578 	fri.fi = fib4_entry->fi;
5579 	fri.tb_id = fib4_entry->tb_id;
5580 	fri.dst = cpu_to_be32(*p_dst);
5581 	fri.dst_len = dst_len;
5582 	fri.dscp = fib4_entry->dscp;
5583 	fri.type = fib4_entry->type;
5584 	fri.offload = false;
5585 	fri.trap = false;
5586 	fri.offload_failed = false;
5587 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5588 }
5589 
5590 #if IS_ENABLED(CONFIG_IPV6)
5591 static void
5592 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5593 				      struct fib6_info **rt_arr,
5594 				      unsigned int nrt6)
5595 {
5596 	int i;
5597 
5598 	/* In IPv6 a multipath route is represented using multiple routes, so
5599 	 * we need to set the flags on all of them.
5600 	 */
5601 	for (i = 0; i < nrt6; i++)
5602 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i],
5603 				       false, false, true);
5604 }
5605 #else
5606 static void
5607 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5608 				      struct fib6_info **rt_arr,
5609 				      unsigned int nrt6)
5610 {
5611 }
5612 #endif
5613 
5614 #if IS_ENABLED(CONFIG_IPV6)
5615 static void
5616 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5617 				 struct mlxsw_sp_fib_entry *fib_entry)
5618 {
5619 	struct mlxsw_sp_fib6_entry *fib6_entry;
5620 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5621 	bool should_offload;
5622 
5623 	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5624 
5625 	/* In IPv6 a multipath route is represented using multiple routes, so
5626 	 * we need to set the flags on all of them.
5627 	 */
5628 	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5629 				  common);
5630 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5631 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5632 				       should_offload, !should_offload, false);
5633 }
5634 #else
5635 static void
5636 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5637 				 struct mlxsw_sp_fib_entry *fib_entry)
5638 {
5639 }
5640 #endif
5641 
5642 #if IS_ENABLED(CONFIG_IPV6)
5643 static void
5644 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5645 				   struct mlxsw_sp_fib_entry *fib_entry)
5646 {
5647 	struct mlxsw_sp_fib6_entry *fib6_entry;
5648 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5649 
5650 	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5651 				  common);
5652 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5653 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5654 				       false, false, false);
5655 }
5656 #else
5657 static void
5658 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5659 				   struct mlxsw_sp_fib_entry *fib_entry)
5660 {
5661 }
5662 #endif
5663 
5664 static void
5665 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5666 				struct mlxsw_sp_fib_entry *fib_entry)
5667 {
5668 	switch (fib_entry->fib_node->fib->proto) {
5669 	case MLXSW_SP_L3_PROTO_IPV4:
5670 		mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
5671 		break;
5672 	case MLXSW_SP_L3_PROTO_IPV6:
5673 		mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
5674 		break;
5675 	}
5676 }
5677 
5678 static void
5679 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5680 				  struct mlxsw_sp_fib_entry *fib_entry)
5681 {
5682 	switch (fib_entry->fib_node->fib->proto) {
5683 	case MLXSW_SP_L3_PROTO_IPV4:
5684 		mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5685 		break;
5686 	case MLXSW_SP_L3_PROTO_IPV6:
5687 		mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5688 		break;
5689 	}
5690 }
5691 
5692 static void
5693 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
5694 				    struct mlxsw_sp_fib_entry *fib_entry,
5695 				    enum mlxsw_reg_ralue_op op)
5696 {
5697 	switch (op) {
5698 	case MLXSW_REG_RALUE_OP_WRITE_WRITE:
5699 		mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
5700 		break;
5701 	case MLXSW_REG_RALUE_OP_WRITE_DELETE:
5702 		mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5703 		break;
5704 	default:
5705 		break;
5706 	}
5707 }
5708 
5709 static void
5710 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
5711 			      const struct mlxsw_sp_fib_entry *fib_entry,
5712 			      enum mlxsw_reg_ralue_op op)
5713 {
5714 	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
5715 	enum mlxsw_reg_ralxx_protocol proto;
5716 	u32 *p_dip;
5717 
5718 	proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
5719 
5720 	switch (fib->proto) {
5721 	case MLXSW_SP_L3_PROTO_IPV4:
5722 		p_dip = (u32 *) fib_entry->fib_node->key.addr;
5723 		mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
5724 				      fib_entry->fib_node->key.prefix_len,
5725 				      *p_dip);
5726 		break;
5727 	case MLXSW_SP_L3_PROTO_IPV6:
5728 		mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
5729 				      fib_entry->fib_node->key.prefix_len,
5730 				      fib_entry->fib_node->key.addr);
5731 		break;
5732 	}
5733 }
5734 
5735 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
5736 					struct mlxsw_sp_fib_entry *fib_entry,
5737 					enum mlxsw_reg_ralue_op op)
5738 {
5739 	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5740 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
5741 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5742 	enum mlxsw_reg_ralue_trap_action trap_action;
5743 	u16 trap_id = 0;
5744 	u32 adjacency_index = 0;
5745 	u16 ecmp_size = 0;
5746 
5747 	/* In case the nexthop group adjacency index is valid, use it
5748 	 * with provided ECMP size. Otherwise, setup trap and pass
5749 	 * traffic to kernel.
5750 	 */
5751 	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5752 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5753 		adjacency_index = nhgi->adj_index;
5754 		ecmp_size = nhgi->ecmp_size;
5755 	} else if (!nhgi->adj_index_valid && nhgi->count && nhgi->nh_rif) {
5756 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5757 		adjacency_index = mlxsw_sp->router->adj_trap_index;
5758 		ecmp_size = 1;
5759 	} else {
5760 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5761 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5762 	}
5763 
5764 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5765 	mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
5766 					adjacency_index, ecmp_size);
5767 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5768 }
5769 
5770 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
5771 				       struct mlxsw_sp_fib_entry *fib_entry,
5772 				       enum mlxsw_reg_ralue_op op)
5773 {
5774 	struct mlxsw_sp_rif *rif = fib_entry->nh_group->nhgi->nh_rif;
5775 	enum mlxsw_reg_ralue_trap_action trap_action;
5776 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5777 	u16 trap_id = 0;
5778 	u16 rif_index = 0;
5779 
5780 	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5781 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5782 		rif_index = rif->rif_index;
5783 	} else {
5784 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5785 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5786 	}
5787 
5788 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5789 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
5790 				       rif_index);
5791 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5792 }
5793 
5794 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
5795 				      struct mlxsw_sp_fib_entry *fib_entry,
5796 				      enum mlxsw_reg_ralue_op op)
5797 {
5798 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5799 
5800 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5801 	mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
5802 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5803 }
5804 
5805 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
5806 					   struct mlxsw_sp_fib_entry *fib_entry,
5807 					   enum mlxsw_reg_ralue_op op)
5808 {
5809 	enum mlxsw_reg_ralue_trap_action trap_action;
5810 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5811 
5812 	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
5813 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5814 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0);
5815 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5816 }
5817 
5818 static int
5819 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
5820 				  struct mlxsw_sp_fib_entry *fib_entry,
5821 				  enum mlxsw_reg_ralue_op op)
5822 {
5823 	enum mlxsw_reg_ralue_trap_action trap_action;
5824 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5825 	u16 trap_id;
5826 
5827 	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5828 	trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
5829 
5830 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5831 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0);
5832 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5833 }
5834 
5835 static int
5836 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
5837 				 struct mlxsw_sp_fib_entry *fib_entry,
5838 				 enum mlxsw_reg_ralue_op op)
5839 {
5840 	struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
5841 	const struct mlxsw_sp_ipip_ops *ipip_ops;
5842 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5843 	int err;
5844 
5845 	if (WARN_ON(!ipip_entry))
5846 		return -EINVAL;
5847 
5848 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
5849 	err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
5850 				     fib_entry->decap.tunnel_index);
5851 	if (err)
5852 		return err;
5853 
5854 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5855 	mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
5856 					   fib_entry->decap.tunnel_index);
5857 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5858 }
5859 
5860 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
5861 					   struct mlxsw_sp_fib_entry *fib_entry,
5862 					   enum mlxsw_reg_ralue_op op)
5863 {
5864 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5865 
5866 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5867 	mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
5868 					   fib_entry->decap.tunnel_index);
5869 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5870 }
5871 
5872 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
5873 				   struct mlxsw_sp_fib_entry *fib_entry,
5874 				   enum mlxsw_reg_ralue_op op)
5875 {
5876 	switch (fib_entry->type) {
5877 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5878 		return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
5879 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5880 		return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
5881 	case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
5882 		return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
5883 	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5884 		return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op);
5885 	case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
5886 		return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry,
5887 							 op);
5888 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5889 		return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
5890 							fib_entry, op);
5891 	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5892 		return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op);
5893 	}
5894 	return -EINVAL;
5895 }
5896 
5897 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
5898 				 struct mlxsw_sp_fib_entry *fib_entry,
5899 				 enum mlxsw_reg_ralue_op op)
5900 {
5901 	int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
5902 
5903 	if (err)
5904 		return err;
5905 
5906 	mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
5907 
5908 	return err;
5909 }
5910 
5911 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
5912 				     struct mlxsw_sp_fib_entry *fib_entry)
5913 {
5914 	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
5915 				     MLXSW_REG_RALUE_OP_WRITE_WRITE);
5916 }
5917 
5918 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
5919 				  struct mlxsw_sp_fib_entry *fib_entry)
5920 {
5921 	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
5922 				     MLXSW_REG_RALUE_OP_WRITE_DELETE);
5923 }
5924 
5925 static int
5926 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5927 			     const struct fib_entry_notifier_info *fen_info,
5928 			     struct mlxsw_sp_fib_entry *fib_entry)
5929 {
5930 	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
5931 	union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
5932 	struct mlxsw_sp_router *router = mlxsw_sp->router;
5933 	u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
5934 	int ifindex = nhgi->nexthops[0].ifindex;
5935 	struct mlxsw_sp_ipip_entry *ipip_entry;
5936 
5937 	switch (fen_info->type) {
5938 	case RTN_LOCAL:
5939 		ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
5940 							       MLXSW_SP_L3_PROTO_IPV4, dip);
5941 		if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
5942 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
5943 			return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
5944 							     fib_entry,
5945 							     ipip_entry);
5946 		}
5947 		if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
5948 						 MLXSW_SP_L3_PROTO_IPV4,
5949 						 &dip)) {
5950 			u32 tunnel_index;
5951 
5952 			tunnel_index = router->nve_decap_config.tunnel_index;
5953 			fib_entry->decap.tunnel_index = tunnel_index;
5954 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
5955 			return 0;
5956 		}
5957 		fallthrough;
5958 	case RTN_BROADCAST:
5959 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5960 		return 0;
5961 	case RTN_BLACKHOLE:
5962 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5963 		return 0;
5964 	case RTN_UNREACHABLE:
5965 	case RTN_PROHIBIT:
5966 		/* Packets hitting these routes need to be trapped, but
5967 		 * can do so with a lower priority than packets directed
5968 		 * at the host, so use action type local instead of trap.
5969 		 */
5970 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
5971 		return 0;
5972 	case RTN_UNICAST:
5973 		if (nhgi->gateway)
5974 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5975 		else
5976 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5977 		return 0;
5978 	default:
5979 		return -EINVAL;
5980 	}
5981 }
5982 
5983 static void
5984 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
5985 			      struct mlxsw_sp_fib_entry *fib_entry)
5986 {
5987 	switch (fib_entry->type) {
5988 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5989 		mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
5990 		break;
5991 	default:
5992 		break;
5993 	}
5994 }
5995 
5996 static void
5997 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
5998 			       struct mlxsw_sp_fib4_entry *fib4_entry)
5999 {
6000 	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common);
6001 }
6002 
6003 static struct mlxsw_sp_fib4_entry *
6004 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
6005 			   struct mlxsw_sp_fib_node *fib_node,
6006 			   const struct fib_entry_notifier_info *fen_info)
6007 {
6008 	struct mlxsw_sp_fib4_entry *fib4_entry;
6009 	struct mlxsw_sp_fib_entry *fib_entry;
6010 	int err;
6011 
6012 	fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
6013 	if (!fib4_entry)
6014 		return ERR_PTR(-ENOMEM);
6015 	fib_entry = &fib4_entry->common;
6016 
6017 	err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
6018 	if (err)
6019 		goto err_nexthop4_group_get;
6020 
6021 	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
6022 					     fib_node->fib);
6023 	if (err)
6024 		goto err_nexthop_group_vr_link;
6025 
6026 	err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
6027 	if (err)
6028 		goto err_fib4_entry_type_set;
6029 
6030 	fib4_entry->fi = fen_info->fi;
6031 	fib_info_hold(fib4_entry->fi);
6032 	fib4_entry->tb_id = fen_info->tb_id;
6033 	fib4_entry->type = fen_info->type;
6034 	fib4_entry->dscp = fen_info->dscp;
6035 
6036 	fib_entry->fib_node = fib_node;
6037 
6038 	return fib4_entry;
6039 
6040 err_fib4_entry_type_set:
6041 	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
6042 err_nexthop_group_vr_link:
6043 	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6044 err_nexthop4_group_get:
6045 	kfree(fib4_entry);
6046 	return ERR_PTR(err);
6047 }
6048 
6049 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6050 					struct mlxsw_sp_fib4_entry *fib4_entry)
6051 {
6052 	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6053 
6054 	fib_info_put(fib4_entry->fi);
6055 	mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry);
6056 	mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group,
6057 					 fib_node->fib);
6058 	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6059 	kfree(fib4_entry);
6060 }
6061 
6062 static struct mlxsw_sp_fib4_entry *
6063 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6064 			   const struct fib_entry_notifier_info *fen_info)
6065 {
6066 	struct mlxsw_sp_fib4_entry *fib4_entry;
6067 	struct mlxsw_sp_fib_node *fib_node;
6068 	struct mlxsw_sp_fib *fib;
6069 	struct mlxsw_sp_vr *vr;
6070 
6071 	vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
6072 	if (!vr)
6073 		return NULL;
6074 	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
6075 
6076 	fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
6077 					    sizeof(fen_info->dst),
6078 					    fen_info->dst_len);
6079 	if (!fib_node)
6080 		return NULL;
6081 
6082 	fib4_entry = container_of(fib_node->fib_entry,
6083 				  struct mlxsw_sp_fib4_entry, common);
6084 	if (fib4_entry->tb_id == fen_info->tb_id &&
6085 	    fib4_entry->dscp == fen_info->dscp &&
6086 	    fib4_entry->type == fen_info->type &&
6087 	    fib4_entry->fi == fen_info->fi)
6088 		return fib4_entry;
6089 
6090 	return NULL;
6091 }
6092 
6093 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
6094 	.key_offset = offsetof(struct mlxsw_sp_fib_node, key),
6095 	.head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
6096 	.key_len = sizeof(struct mlxsw_sp_fib_key),
6097 	.automatic_shrinking = true,
6098 };
6099 
6100 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
6101 				    struct mlxsw_sp_fib_node *fib_node)
6102 {
6103 	return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
6104 				      mlxsw_sp_fib_ht_params);
6105 }
6106 
6107 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
6108 				     struct mlxsw_sp_fib_node *fib_node)
6109 {
6110 	rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
6111 			       mlxsw_sp_fib_ht_params);
6112 }
6113 
6114 static struct mlxsw_sp_fib_node *
6115 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
6116 			 size_t addr_len, unsigned char prefix_len)
6117 {
6118 	struct mlxsw_sp_fib_key key;
6119 
6120 	memset(&key, 0, sizeof(key));
6121 	memcpy(key.addr, addr, addr_len);
6122 	key.prefix_len = prefix_len;
6123 	return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
6124 }
6125 
6126 static struct mlxsw_sp_fib_node *
6127 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
6128 			 size_t addr_len, unsigned char prefix_len)
6129 {
6130 	struct mlxsw_sp_fib_node *fib_node;
6131 
6132 	fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
6133 	if (!fib_node)
6134 		return NULL;
6135 
6136 	list_add(&fib_node->list, &fib->node_list);
6137 	memcpy(fib_node->key.addr, addr, addr_len);
6138 	fib_node->key.prefix_len = prefix_len;
6139 
6140 	return fib_node;
6141 }
6142 
6143 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
6144 {
6145 	list_del(&fib_node->list);
6146 	kfree(fib_node);
6147 }
6148 
6149 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
6150 				      struct mlxsw_sp_fib_node *fib_node)
6151 {
6152 	struct mlxsw_sp_prefix_usage req_prefix_usage;
6153 	struct mlxsw_sp_fib *fib = fib_node->fib;
6154 	struct mlxsw_sp_lpm_tree *lpm_tree;
6155 	int err;
6156 
6157 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
6158 	if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6159 		goto out;
6160 
6161 	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6162 	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
6163 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6164 					 fib->proto);
6165 	if (IS_ERR(lpm_tree))
6166 		return PTR_ERR(lpm_tree);
6167 
6168 	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6169 	if (err)
6170 		goto err_lpm_tree_replace;
6171 
6172 out:
6173 	lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
6174 	return 0;
6175 
6176 err_lpm_tree_replace:
6177 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6178 	return err;
6179 }
6180 
6181 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
6182 					 struct mlxsw_sp_fib_node *fib_node)
6183 {
6184 	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
6185 	struct mlxsw_sp_prefix_usage req_prefix_usage;
6186 	struct mlxsw_sp_fib *fib = fib_node->fib;
6187 	int err;
6188 
6189 	if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6190 		return;
6191 	/* Try to construct a new LPM tree from the current prefix usage
6192 	 * minus the unused one. If we fail, continue using the old one.
6193 	 */
6194 	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6195 	mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
6196 				    fib_node->key.prefix_len);
6197 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6198 					 fib->proto);
6199 	if (IS_ERR(lpm_tree))
6200 		return;
6201 
6202 	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6203 	if (err)
6204 		goto err_lpm_tree_replace;
6205 
6206 	return;
6207 
6208 err_lpm_tree_replace:
6209 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6210 }
6211 
6212 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
6213 				  struct mlxsw_sp_fib_node *fib_node,
6214 				  struct mlxsw_sp_fib *fib)
6215 {
6216 	int err;
6217 
6218 	err = mlxsw_sp_fib_node_insert(fib, fib_node);
6219 	if (err)
6220 		return err;
6221 	fib_node->fib = fib;
6222 
6223 	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
6224 	if (err)
6225 		goto err_fib_lpm_tree_link;
6226 
6227 	return 0;
6228 
6229 err_fib_lpm_tree_link:
6230 	fib_node->fib = NULL;
6231 	mlxsw_sp_fib_node_remove(fib, fib_node);
6232 	return err;
6233 }
6234 
6235 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
6236 				   struct mlxsw_sp_fib_node *fib_node)
6237 {
6238 	struct mlxsw_sp_fib *fib = fib_node->fib;
6239 
6240 	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
6241 	fib_node->fib = NULL;
6242 	mlxsw_sp_fib_node_remove(fib, fib_node);
6243 }
6244 
6245 static struct mlxsw_sp_fib_node *
6246 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
6247 		      size_t addr_len, unsigned char prefix_len,
6248 		      enum mlxsw_sp_l3proto proto)
6249 {
6250 	struct mlxsw_sp_fib_node *fib_node;
6251 	struct mlxsw_sp_fib *fib;
6252 	struct mlxsw_sp_vr *vr;
6253 	int err;
6254 
6255 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
6256 	if (IS_ERR(vr))
6257 		return ERR_CAST(vr);
6258 	fib = mlxsw_sp_vr_fib(vr, proto);
6259 
6260 	fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
6261 	if (fib_node)
6262 		return fib_node;
6263 
6264 	fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
6265 	if (!fib_node) {
6266 		err = -ENOMEM;
6267 		goto err_fib_node_create;
6268 	}
6269 
6270 	err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
6271 	if (err)
6272 		goto err_fib_node_init;
6273 
6274 	return fib_node;
6275 
6276 err_fib_node_init:
6277 	mlxsw_sp_fib_node_destroy(fib_node);
6278 err_fib_node_create:
6279 	mlxsw_sp_vr_put(mlxsw_sp, vr);
6280 	return ERR_PTR(err);
6281 }
6282 
6283 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
6284 				  struct mlxsw_sp_fib_node *fib_node)
6285 {
6286 	struct mlxsw_sp_vr *vr = fib_node->fib->vr;
6287 
6288 	if (fib_node->fib_entry)
6289 		return;
6290 	mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
6291 	mlxsw_sp_fib_node_destroy(fib_node);
6292 	mlxsw_sp_vr_put(mlxsw_sp, vr);
6293 }
6294 
6295 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
6296 					struct mlxsw_sp_fib_entry *fib_entry)
6297 {
6298 	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6299 	int err;
6300 
6301 	fib_node->fib_entry = fib_entry;
6302 
6303 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
6304 	if (err)
6305 		goto err_fib_entry_update;
6306 
6307 	return 0;
6308 
6309 err_fib_entry_update:
6310 	fib_node->fib_entry = NULL;
6311 	return err;
6312 }
6313 
6314 static void
6315 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
6316 			       struct mlxsw_sp_fib_entry *fib_entry)
6317 {
6318 	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6319 
6320 	mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
6321 	fib_node->fib_entry = NULL;
6322 }
6323 
6324 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
6325 {
6326 	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6327 	struct mlxsw_sp_fib4_entry *fib4_replaced;
6328 
6329 	if (!fib_node->fib_entry)
6330 		return true;
6331 
6332 	fib4_replaced = container_of(fib_node->fib_entry,
6333 				     struct mlxsw_sp_fib4_entry, common);
6334 	if (fib4_entry->tb_id == RT_TABLE_MAIN &&
6335 	    fib4_replaced->tb_id == RT_TABLE_LOCAL)
6336 		return false;
6337 
6338 	return true;
6339 }
6340 
6341 static int
6342 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
6343 			     const struct fib_entry_notifier_info *fen_info)
6344 {
6345 	struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
6346 	struct mlxsw_sp_fib_entry *replaced;
6347 	struct mlxsw_sp_fib_node *fib_node;
6348 	int err;
6349 
6350 	if (fen_info->fi->nh &&
6351 	    !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id))
6352 		return 0;
6353 
6354 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
6355 					 &fen_info->dst, sizeof(fen_info->dst),
6356 					 fen_info->dst_len,
6357 					 MLXSW_SP_L3_PROTO_IPV4);
6358 	if (IS_ERR(fib_node)) {
6359 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
6360 		return PTR_ERR(fib_node);
6361 	}
6362 
6363 	fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
6364 	if (IS_ERR(fib4_entry)) {
6365 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
6366 		err = PTR_ERR(fib4_entry);
6367 		goto err_fib4_entry_create;
6368 	}
6369 
6370 	if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
6371 		mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6372 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6373 		return 0;
6374 	}
6375 
6376 	replaced = fib_node->fib_entry;
6377 	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common);
6378 	if (err) {
6379 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
6380 		goto err_fib_node_entry_link;
6381 	}
6382 
6383 	/* Nothing to replace */
6384 	if (!replaced)
6385 		return 0;
6386 
6387 	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
6388 	fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
6389 				     common);
6390 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
6391 
6392 	return 0;
6393 
6394 err_fib_node_entry_link:
6395 	fib_node->fib_entry = replaced;
6396 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6397 err_fib4_entry_create:
6398 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6399 	return err;
6400 }
6401 
6402 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
6403 				     struct fib_entry_notifier_info *fen_info)
6404 {
6405 	struct mlxsw_sp_fib4_entry *fib4_entry;
6406 	struct mlxsw_sp_fib_node *fib_node;
6407 
6408 	fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
6409 	if (!fib4_entry)
6410 		return;
6411 	fib_node = fib4_entry->common.fib_node;
6412 
6413 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common);
6414 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6415 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6416 }
6417 
6418 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
6419 {
6420 	/* Multicast routes aren't supported, so ignore them. Neighbour
6421 	 * Discovery packets are specifically trapped.
6422 	 */
6423 	if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
6424 		return true;
6425 
6426 	/* Cloned routes are irrelevant in the forwarding path. */
6427 	if (rt->fib6_flags & RTF_CACHE)
6428 		return true;
6429 
6430 	return false;
6431 }
6432 
6433 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
6434 {
6435 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6436 
6437 	mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
6438 	if (!mlxsw_sp_rt6)
6439 		return ERR_PTR(-ENOMEM);
6440 
6441 	/* In case of route replace, replaced route is deleted with
6442 	 * no notification. Take reference to prevent accessing freed
6443 	 * memory.
6444 	 */
6445 	mlxsw_sp_rt6->rt = rt;
6446 	fib6_info_hold(rt);
6447 
6448 	return mlxsw_sp_rt6;
6449 }
6450 
6451 #if IS_ENABLED(CONFIG_IPV6)
6452 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6453 {
6454 	fib6_info_release(rt);
6455 }
6456 #else
6457 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6458 {
6459 }
6460 #endif
6461 
6462 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
6463 {
6464 	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
6465 
6466 	if (!mlxsw_sp_rt6->rt->nh)
6467 		fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
6468 	mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
6469 	kfree(mlxsw_sp_rt6);
6470 }
6471 
6472 static struct fib6_info *
6473 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
6474 {
6475 	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
6476 				list)->rt;
6477 }
6478 
6479 static struct mlxsw_sp_rt6 *
6480 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
6481 			    const struct fib6_info *rt)
6482 {
6483 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6484 
6485 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
6486 		if (mlxsw_sp_rt6->rt == rt)
6487 			return mlxsw_sp_rt6;
6488 	}
6489 
6490 	return NULL;
6491 }
6492 
6493 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
6494 					const struct fib6_info *rt,
6495 					enum mlxsw_sp_ipip_type *ret)
6496 {
6497 	return rt->fib6_nh->fib_nh_dev &&
6498 	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
6499 }
6500 
6501 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
6502 				  struct mlxsw_sp_nexthop_group *nh_grp,
6503 				  struct mlxsw_sp_nexthop *nh,
6504 				  const struct fib6_info *rt)
6505 {
6506 	struct net_device *dev = rt->fib6_nh->fib_nh_dev;
6507 	int err;
6508 
6509 	nh->nhgi = nh_grp->nhgi;
6510 	nh->nh_weight = rt->fib6_nh->fib_nh_weight;
6511 	memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
6512 #if IS_ENABLED(CONFIG_IPV6)
6513 	nh->neigh_tbl = &nd_tbl;
6514 #endif
6515 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
6516 
6517 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
6518 
6519 	if (!dev)
6520 		return 0;
6521 	nh->ifindex = dev->ifindex;
6522 
6523 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
6524 	if (err)
6525 		goto err_nexthop_type_init;
6526 
6527 	return 0;
6528 
6529 err_nexthop_type_init:
6530 	list_del(&nh->router_list_node);
6531 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6532 	return err;
6533 }
6534 
6535 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
6536 				   struct mlxsw_sp_nexthop *nh)
6537 {
6538 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
6539 	list_del(&nh->router_list_node);
6540 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6541 }
6542 
6543 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
6544 				    const struct fib6_info *rt)
6545 {
6546 	return rt->fib6_nh->fib_nh_gw_family ||
6547 	       mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
6548 }
6549 
6550 static int
6551 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
6552 				  struct mlxsw_sp_nexthop_group *nh_grp,
6553 				  struct mlxsw_sp_fib6_entry *fib6_entry)
6554 {
6555 	struct mlxsw_sp_nexthop_group_info *nhgi;
6556 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6557 	struct mlxsw_sp_nexthop *nh;
6558 	int err, i;
6559 
6560 	nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
6561 		       GFP_KERNEL);
6562 	if (!nhgi)
6563 		return -ENOMEM;
6564 	nh_grp->nhgi = nhgi;
6565 	nhgi->nh_grp = nh_grp;
6566 	mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
6567 					struct mlxsw_sp_rt6, list);
6568 	nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
6569 	nhgi->count = fib6_entry->nrt6;
6570 	for (i = 0; i < nhgi->count; i++) {
6571 		struct fib6_info *rt = mlxsw_sp_rt6->rt;
6572 
6573 		nh = &nhgi->nexthops[i];
6574 		err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
6575 		if (err)
6576 			goto err_nexthop6_init;
6577 		mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
6578 	}
6579 	nh_grp->nhgi = nhgi;
6580 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
6581 	if (err)
6582 		goto err_group_inc;
6583 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6584 	if (err)
6585 		goto err_group_refresh;
6586 
6587 	return 0;
6588 
6589 err_group_refresh:
6590 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6591 err_group_inc:
6592 	i = nhgi->count;
6593 err_nexthop6_init:
6594 	for (i--; i >= 0; i--) {
6595 		nh = &nhgi->nexthops[i];
6596 		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6597 	}
6598 	kfree(nhgi);
6599 	return err;
6600 }
6601 
6602 static void
6603 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
6604 				  struct mlxsw_sp_nexthop_group *nh_grp)
6605 {
6606 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
6607 	int i;
6608 
6609 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6610 	for (i = nhgi->count - 1; i >= 0; i--) {
6611 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
6612 
6613 		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6614 	}
6615 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6616 	WARN_ON_ONCE(nhgi->adj_index_valid);
6617 	kfree(nhgi);
6618 }
6619 
6620 static struct mlxsw_sp_nexthop_group *
6621 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
6622 			       struct mlxsw_sp_fib6_entry *fib6_entry)
6623 {
6624 	struct mlxsw_sp_nexthop_group *nh_grp;
6625 	int err;
6626 
6627 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
6628 	if (!nh_grp)
6629 		return ERR_PTR(-ENOMEM);
6630 	INIT_LIST_HEAD(&nh_grp->vr_list);
6631 	err = rhashtable_init(&nh_grp->vr_ht,
6632 			      &mlxsw_sp_nexthop_group_vr_ht_params);
6633 	if (err)
6634 		goto err_nexthop_group_vr_ht_init;
6635 	INIT_LIST_HEAD(&nh_grp->fib_list);
6636 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
6637 
6638 	err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
6639 	if (err)
6640 		goto err_nexthop_group_info_init;
6641 
6642 	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
6643 	if (err)
6644 		goto err_nexthop_group_insert;
6645 
6646 	nh_grp->can_destroy = true;
6647 
6648 	return nh_grp;
6649 
6650 err_nexthop_group_insert:
6651 	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6652 err_nexthop_group_info_init:
6653 	rhashtable_destroy(&nh_grp->vr_ht);
6654 err_nexthop_group_vr_ht_init:
6655 	kfree(nh_grp);
6656 	return ERR_PTR(err);
6657 }
6658 
6659 static void
6660 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
6661 				struct mlxsw_sp_nexthop_group *nh_grp)
6662 {
6663 	if (!nh_grp->can_destroy)
6664 		return;
6665 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
6666 	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6667 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
6668 	rhashtable_destroy(&nh_grp->vr_ht);
6669 	kfree(nh_grp);
6670 }
6671 
6672 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
6673 				       struct mlxsw_sp_fib6_entry *fib6_entry)
6674 {
6675 	struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6676 	struct mlxsw_sp_nexthop_group *nh_grp;
6677 
6678 	if (rt->nh) {
6679 		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
6680 							   rt->nh->id);
6681 		if (WARN_ON_ONCE(!nh_grp))
6682 			return -EINVAL;
6683 		goto out;
6684 	}
6685 
6686 	nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
6687 	if (!nh_grp) {
6688 		nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
6689 		if (IS_ERR(nh_grp))
6690 			return PTR_ERR(nh_grp);
6691 	}
6692 
6693 	/* The route and the nexthop are described by the same struct, so we
6694 	 * need to the update the nexthop offload indication for the new route.
6695 	 */
6696 	__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
6697 
6698 out:
6699 	list_add_tail(&fib6_entry->common.nexthop_group_node,
6700 		      &nh_grp->fib_list);
6701 	fib6_entry->common.nh_group = nh_grp;
6702 
6703 	return 0;
6704 }
6705 
6706 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
6707 					struct mlxsw_sp_fib_entry *fib_entry)
6708 {
6709 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
6710 
6711 	list_del(&fib_entry->nexthop_group_node);
6712 	if (!list_empty(&nh_grp->fib_list))
6713 		return;
6714 
6715 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
6716 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
6717 		return;
6718 	}
6719 
6720 	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
6721 }
6722 
6723 static int
6724 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
6725 			       struct mlxsw_sp_fib6_entry *fib6_entry)
6726 {
6727 	struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
6728 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6729 	int err;
6730 
6731 	mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib);
6732 	fib6_entry->common.nh_group = NULL;
6733 	list_del(&fib6_entry->common.nexthop_group_node);
6734 
6735 	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6736 	if (err)
6737 		goto err_nexthop6_group_get;
6738 
6739 	err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group,
6740 					     fib_node->fib);
6741 	if (err)
6742 		goto err_nexthop_group_vr_link;
6743 
6744 	/* In case this entry is offloaded, then the adjacency index
6745 	 * currently associated with it in the device's table is that
6746 	 * of the old group. Start using the new one instead.
6747 	 */
6748 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common);
6749 	if (err)
6750 		goto err_fib_entry_update;
6751 
6752 	if (list_empty(&old_nh_grp->fib_list))
6753 		mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
6754 
6755 	return 0;
6756 
6757 err_fib_entry_update:
6758 	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6759 					 fib_node->fib);
6760 err_nexthop_group_vr_link:
6761 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6762 err_nexthop6_group_get:
6763 	list_add_tail(&fib6_entry->common.nexthop_group_node,
6764 		      &old_nh_grp->fib_list);
6765 	fib6_entry->common.nh_group = old_nh_grp;
6766 	mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib);
6767 	return err;
6768 }
6769 
6770 static int
6771 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
6772 				struct mlxsw_sp_fib6_entry *fib6_entry,
6773 				struct fib6_info **rt_arr, unsigned int nrt6)
6774 {
6775 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6776 	int err, i;
6777 
6778 	for (i = 0; i < nrt6; i++) {
6779 		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
6780 		if (IS_ERR(mlxsw_sp_rt6)) {
6781 			err = PTR_ERR(mlxsw_sp_rt6);
6782 			goto err_rt6_unwind;
6783 		}
6784 
6785 		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
6786 		fib6_entry->nrt6++;
6787 	}
6788 
6789 	err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
6790 	if (err)
6791 		goto err_rt6_unwind;
6792 
6793 	return 0;
6794 
6795 err_rt6_unwind:
6796 	for (; i > 0; i--) {
6797 		fib6_entry->nrt6--;
6798 		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
6799 					       struct mlxsw_sp_rt6, list);
6800 		list_del(&mlxsw_sp_rt6->list);
6801 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6802 	}
6803 	return err;
6804 }
6805 
6806 static void
6807 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
6808 				struct mlxsw_sp_fib6_entry *fib6_entry,
6809 				struct fib6_info **rt_arr, unsigned int nrt6)
6810 {
6811 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6812 	int i;
6813 
6814 	for (i = 0; i < nrt6; i++) {
6815 		mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
6816 							   rt_arr[i]);
6817 		if (WARN_ON_ONCE(!mlxsw_sp_rt6))
6818 			continue;
6819 
6820 		fib6_entry->nrt6--;
6821 		list_del(&mlxsw_sp_rt6->list);
6822 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6823 	}
6824 
6825 	mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
6826 }
6827 
6828 static int
6829 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp,
6830 				   struct mlxsw_sp_fib_entry *fib_entry,
6831 				   const struct fib6_info *rt)
6832 {
6833 	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
6834 	union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr };
6835 	u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id);
6836 	struct mlxsw_sp_router *router = mlxsw_sp->router;
6837 	int ifindex = nhgi->nexthops[0].ifindex;
6838 	struct mlxsw_sp_ipip_entry *ipip_entry;
6839 
6840 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6841 	ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
6842 						       MLXSW_SP_L3_PROTO_IPV6,
6843 						       dip);
6844 
6845 	if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
6846 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
6847 		return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry,
6848 						     ipip_entry);
6849 	}
6850 	if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
6851 					 MLXSW_SP_L3_PROTO_IPV6, &dip)) {
6852 		u32 tunnel_index;
6853 
6854 		tunnel_index = router->nve_decap_config.tunnel_index;
6855 		fib_entry->decap.tunnel_index = tunnel_index;
6856 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
6857 	}
6858 
6859 	return 0;
6860 }
6861 
6862 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
6863 					struct mlxsw_sp_fib_entry *fib_entry,
6864 					const struct fib6_info *rt)
6865 {
6866 	if (rt->fib6_flags & RTF_LOCAL)
6867 		return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry,
6868 							  rt);
6869 	if (rt->fib6_flags & RTF_ANYCAST)
6870 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6871 	else if (rt->fib6_type == RTN_BLACKHOLE)
6872 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
6873 	else if (rt->fib6_flags & RTF_REJECT)
6874 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
6875 	else if (fib_entry->nh_group->nhgi->gateway)
6876 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
6877 	else
6878 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
6879 
6880 	return 0;
6881 }
6882 
6883 static void
6884 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
6885 {
6886 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
6887 
6888 	list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
6889 				 list) {
6890 		fib6_entry->nrt6--;
6891 		list_del(&mlxsw_sp_rt6->list);
6892 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6893 	}
6894 }
6895 
6896 static struct mlxsw_sp_fib6_entry *
6897 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
6898 			   struct mlxsw_sp_fib_node *fib_node,
6899 			   struct fib6_info **rt_arr, unsigned int nrt6)
6900 {
6901 	struct mlxsw_sp_fib6_entry *fib6_entry;
6902 	struct mlxsw_sp_fib_entry *fib_entry;
6903 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6904 	int err, i;
6905 
6906 	fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
6907 	if (!fib6_entry)
6908 		return ERR_PTR(-ENOMEM);
6909 	fib_entry = &fib6_entry->common;
6910 
6911 	INIT_LIST_HEAD(&fib6_entry->rt6_list);
6912 
6913 	for (i = 0; i < nrt6; i++) {
6914 		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
6915 		if (IS_ERR(mlxsw_sp_rt6)) {
6916 			err = PTR_ERR(mlxsw_sp_rt6);
6917 			goto err_rt6_unwind;
6918 		}
6919 		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
6920 		fib6_entry->nrt6++;
6921 	}
6922 
6923 	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6924 	if (err)
6925 		goto err_rt6_unwind;
6926 
6927 	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
6928 					     fib_node->fib);
6929 	if (err)
6930 		goto err_nexthop_group_vr_link;
6931 
6932 	err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
6933 	if (err)
6934 		goto err_fib6_entry_type_set;
6935 
6936 	fib_entry->fib_node = fib_node;
6937 
6938 	return fib6_entry;
6939 
6940 err_fib6_entry_type_set:
6941 	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
6942 err_nexthop_group_vr_link:
6943 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
6944 err_rt6_unwind:
6945 	for (; i > 0; i--) {
6946 		fib6_entry->nrt6--;
6947 		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
6948 					       struct mlxsw_sp_rt6, list);
6949 		list_del(&mlxsw_sp_rt6->list);
6950 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
6951 	}
6952 	kfree(fib6_entry);
6953 	return ERR_PTR(err);
6954 }
6955 
6956 static void
6957 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6958 			       struct mlxsw_sp_fib6_entry *fib6_entry)
6959 {
6960 	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common);
6961 }
6962 
6963 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6964 					struct mlxsw_sp_fib6_entry *fib6_entry)
6965 {
6966 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6967 
6968 	mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry);
6969 	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6970 					 fib_node->fib);
6971 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6972 	mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
6973 	WARN_ON(fib6_entry->nrt6);
6974 	kfree(fib6_entry);
6975 }
6976 
6977 static struct mlxsw_sp_fib6_entry *
6978 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6979 			   const struct fib6_info *rt)
6980 {
6981 	struct mlxsw_sp_fib6_entry *fib6_entry;
6982 	struct mlxsw_sp_fib_node *fib_node;
6983 	struct mlxsw_sp_fib *fib;
6984 	struct fib6_info *cmp_rt;
6985 	struct mlxsw_sp_vr *vr;
6986 
6987 	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
6988 	if (!vr)
6989 		return NULL;
6990 	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
6991 
6992 	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
6993 					    sizeof(rt->fib6_dst.addr),
6994 					    rt->fib6_dst.plen);
6995 	if (!fib_node)
6996 		return NULL;
6997 
6998 	fib6_entry = container_of(fib_node->fib_entry,
6999 				  struct mlxsw_sp_fib6_entry, common);
7000 	cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7001 	if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
7002 	    rt->fib6_metric == cmp_rt->fib6_metric &&
7003 	    mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
7004 		return fib6_entry;
7005 
7006 	return NULL;
7007 }
7008 
7009 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
7010 {
7011 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7012 	struct mlxsw_sp_fib6_entry *fib6_replaced;
7013 	struct fib6_info *rt, *rt_replaced;
7014 
7015 	if (!fib_node->fib_entry)
7016 		return true;
7017 
7018 	fib6_replaced = container_of(fib_node->fib_entry,
7019 				     struct mlxsw_sp_fib6_entry,
7020 				     common);
7021 	rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7022 	rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
7023 	if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
7024 	    rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
7025 		return false;
7026 
7027 	return true;
7028 }
7029 
7030 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
7031 					struct fib6_info **rt_arr,
7032 					unsigned int nrt6)
7033 {
7034 	struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
7035 	struct mlxsw_sp_fib_entry *replaced;
7036 	struct mlxsw_sp_fib_node *fib_node;
7037 	struct fib6_info *rt = rt_arr[0];
7038 	int err;
7039 
7040 	if (rt->fib6_src.plen)
7041 		return -EINVAL;
7042 
7043 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7044 		return 0;
7045 
7046 	if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id))
7047 		return 0;
7048 
7049 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7050 					 &rt->fib6_dst.addr,
7051 					 sizeof(rt->fib6_dst.addr),
7052 					 rt->fib6_dst.plen,
7053 					 MLXSW_SP_L3_PROTO_IPV6);
7054 	if (IS_ERR(fib_node))
7055 		return PTR_ERR(fib_node);
7056 
7057 	fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
7058 						nrt6);
7059 	if (IS_ERR(fib6_entry)) {
7060 		err = PTR_ERR(fib6_entry);
7061 		goto err_fib6_entry_create;
7062 	}
7063 
7064 	if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
7065 		mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7066 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7067 		return 0;
7068 	}
7069 
7070 	replaced = fib_node->fib_entry;
7071 	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common);
7072 	if (err)
7073 		goto err_fib_node_entry_link;
7074 
7075 	/* Nothing to replace */
7076 	if (!replaced)
7077 		return 0;
7078 
7079 	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
7080 	fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
7081 				     common);
7082 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
7083 
7084 	return 0;
7085 
7086 err_fib_node_entry_link:
7087 	fib_node->fib_entry = replaced;
7088 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7089 err_fib6_entry_create:
7090 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7091 	return err;
7092 }
7093 
7094 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
7095 				       struct fib6_info **rt_arr,
7096 				       unsigned int nrt6)
7097 {
7098 	struct mlxsw_sp_fib6_entry *fib6_entry;
7099 	struct mlxsw_sp_fib_node *fib_node;
7100 	struct fib6_info *rt = rt_arr[0];
7101 	int err;
7102 
7103 	if (rt->fib6_src.plen)
7104 		return -EINVAL;
7105 
7106 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7107 		return 0;
7108 
7109 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7110 					 &rt->fib6_dst.addr,
7111 					 sizeof(rt->fib6_dst.addr),
7112 					 rt->fib6_dst.plen,
7113 					 MLXSW_SP_L3_PROTO_IPV6);
7114 	if (IS_ERR(fib_node))
7115 		return PTR_ERR(fib_node);
7116 
7117 	if (WARN_ON_ONCE(!fib_node->fib_entry)) {
7118 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7119 		return -EINVAL;
7120 	}
7121 
7122 	fib6_entry = container_of(fib_node->fib_entry,
7123 				  struct mlxsw_sp_fib6_entry, common);
7124 	err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr,
7125 					      nrt6);
7126 	if (err)
7127 		goto err_fib6_entry_nexthop_add;
7128 
7129 	return 0;
7130 
7131 err_fib6_entry_nexthop_add:
7132 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7133 	return err;
7134 }
7135 
7136 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
7137 				     struct fib6_info **rt_arr,
7138 				     unsigned int nrt6)
7139 {
7140 	struct mlxsw_sp_fib6_entry *fib6_entry;
7141 	struct mlxsw_sp_fib_node *fib_node;
7142 	struct fib6_info *rt = rt_arr[0];
7143 
7144 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7145 		return;
7146 
7147 	/* Multipath routes are first added to the FIB trie and only then
7148 	 * notified. If we vetoed the addition, we will get a delete
7149 	 * notification for a route we do not have. Therefore, do not warn if
7150 	 * route was not found.
7151 	 */
7152 	fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
7153 	if (!fib6_entry)
7154 		return;
7155 
7156 	/* If not all the nexthops are deleted, then only reduce the nexthop
7157 	 * group.
7158 	 */
7159 	if (nrt6 != fib6_entry->nrt6) {
7160 		mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr,
7161 						nrt6);
7162 		return;
7163 	}
7164 
7165 	fib_node = fib6_entry->common.fib_node;
7166 
7167 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common);
7168 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7169 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7170 }
7171 
7172 static struct mlxsw_sp_mr_table *
7173 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
7174 {
7175 	if (family == RTNL_FAMILY_IPMR)
7176 		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
7177 	else
7178 		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
7179 }
7180 
7181 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
7182 				     struct mfc_entry_notifier_info *men_info,
7183 				     bool replace)
7184 {
7185 	struct mlxsw_sp_mr_table *mrt;
7186 	struct mlxsw_sp_vr *vr;
7187 
7188 	vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
7189 	if (IS_ERR(vr))
7190 		return PTR_ERR(vr);
7191 
7192 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7193 	return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
7194 }
7195 
7196 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
7197 				      struct mfc_entry_notifier_info *men_info)
7198 {
7199 	struct mlxsw_sp_mr_table *mrt;
7200 	struct mlxsw_sp_vr *vr;
7201 
7202 	vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
7203 	if (WARN_ON(!vr))
7204 		return;
7205 
7206 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7207 	mlxsw_sp_mr_route_del(mrt, men_info->mfc);
7208 	mlxsw_sp_vr_put(mlxsw_sp, vr);
7209 }
7210 
7211 static int
7212 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
7213 			      struct vif_entry_notifier_info *ven_info)
7214 {
7215 	struct mlxsw_sp_mr_table *mrt;
7216 	struct mlxsw_sp_rif *rif;
7217 	struct mlxsw_sp_vr *vr;
7218 
7219 	vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
7220 	if (IS_ERR(vr))
7221 		return PTR_ERR(vr);
7222 
7223 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7224 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
7225 	return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
7226 				   ven_info->vif_index,
7227 				   ven_info->vif_flags, rif);
7228 }
7229 
7230 static void
7231 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
7232 			      struct vif_entry_notifier_info *ven_info)
7233 {
7234 	struct mlxsw_sp_mr_table *mrt;
7235 	struct mlxsw_sp_vr *vr;
7236 
7237 	vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
7238 	if (WARN_ON(!vr))
7239 		return;
7240 
7241 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7242 	mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
7243 	mlxsw_sp_vr_put(mlxsw_sp, vr);
7244 }
7245 
7246 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
7247 				     struct mlxsw_sp_fib_node *fib_node)
7248 {
7249 	struct mlxsw_sp_fib4_entry *fib4_entry;
7250 
7251 	fib4_entry = container_of(fib_node->fib_entry,
7252 				  struct mlxsw_sp_fib4_entry, common);
7253 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7254 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
7255 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7256 }
7257 
7258 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
7259 				     struct mlxsw_sp_fib_node *fib_node)
7260 {
7261 	struct mlxsw_sp_fib6_entry *fib6_entry;
7262 
7263 	fib6_entry = container_of(fib_node->fib_entry,
7264 				  struct mlxsw_sp_fib6_entry, common);
7265 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7266 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7267 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7268 }
7269 
7270 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
7271 				    struct mlxsw_sp_fib_node *fib_node)
7272 {
7273 	switch (fib_node->fib->proto) {
7274 	case MLXSW_SP_L3_PROTO_IPV4:
7275 		mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
7276 		break;
7277 	case MLXSW_SP_L3_PROTO_IPV6:
7278 		mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
7279 		break;
7280 	}
7281 }
7282 
7283 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
7284 				  struct mlxsw_sp_vr *vr,
7285 				  enum mlxsw_sp_l3proto proto)
7286 {
7287 	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
7288 	struct mlxsw_sp_fib_node *fib_node, *tmp;
7289 
7290 	list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
7291 		bool do_break = &tmp->list == &fib->node_list;
7292 
7293 		mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
7294 		if (do_break)
7295 			break;
7296 	}
7297 }
7298 
7299 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
7300 {
7301 	int i, j;
7302 
7303 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
7304 		struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
7305 
7306 		if (!mlxsw_sp_vr_is_used(vr))
7307 			continue;
7308 
7309 		for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
7310 			mlxsw_sp_mr_table_flush(vr->mr_table[j]);
7311 		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
7312 
7313 		/* If virtual router was only used for IPv4, then it's no
7314 		 * longer used.
7315 		 */
7316 		if (!mlxsw_sp_vr_is_used(vr))
7317 			continue;
7318 		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
7319 	}
7320 }
7321 
7322 struct mlxsw_sp_fib6_event_work {
7323 	struct fib6_info **rt_arr;
7324 	unsigned int nrt6;
7325 };
7326 
7327 struct mlxsw_sp_fib_event_work {
7328 	struct work_struct work;
7329 	union {
7330 		struct mlxsw_sp_fib6_event_work fib6_work;
7331 		struct fib_entry_notifier_info fen_info;
7332 		struct fib_rule_notifier_info fr_info;
7333 		struct fib_nh_notifier_info fnh_info;
7334 		struct mfc_entry_notifier_info men_info;
7335 		struct vif_entry_notifier_info ven_info;
7336 	};
7337 	struct mlxsw_sp *mlxsw_sp;
7338 	unsigned long event;
7339 };
7340 
7341 static int
7342 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work,
7343 			       struct fib6_entry_notifier_info *fen6_info)
7344 {
7345 	struct fib6_info *rt = fen6_info->rt;
7346 	struct fib6_info **rt_arr;
7347 	struct fib6_info *iter;
7348 	unsigned int nrt6;
7349 	int i = 0;
7350 
7351 	nrt6 = fen6_info->nsiblings + 1;
7352 
7353 	rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
7354 	if (!rt_arr)
7355 		return -ENOMEM;
7356 
7357 	fib6_work->rt_arr = rt_arr;
7358 	fib6_work->nrt6 = nrt6;
7359 
7360 	rt_arr[0] = rt;
7361 	fib6_info_hold(rt);
7362 
7363 	if (!fen6_info->nsiblings)
7364 		return 0;
7365 
7366 	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
7367 		if (i == fen6_info->nsiblings)
7368 			break;
7369 
7370 		rt_arr[i + 1] = iter;
7371 		fib6_info_hold(iter);
7372 		i++;
7373 	}
7374 	WARN_ON_ONCE(i != fen6_info->nsiblings);
7375 
7376 	return 0;
7377 }
7378 
7379 static void
7380 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work)
7381 {
7382 	int i;
7383 
7384 	for (i = 0; i < fib6_work->nrt6; i++)
7385 		mlxsw_sp_rt6_release(fib6_work->rt_arr[i]);
7386 	kfree(fib6_work->rt_arr);
7387 }
7388 
7389 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
7390 {
7391 	struct mlxsw_sp_fib_event_work *fib_work =
7392 		container_of(work, struct mlxsw_sp_fib_event_work, work);
7393 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7394 	int err;
7395 
7396 	mutex_lock(&mlxsw_sp->router->lock);
7397 	mlxsw_sp_span_respin(mlxsw_sp);
7398 
7399 	switch (fib_work->event) {
7400 	case FIB_EVENT_ENTRY_REPLACE:
7401 		err = mlxsw_sp_router_fib4_replace(mlxsw_sp,
7402 						   &fib_work->fen_info);
7403 		if (err) {
7404 			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7405 			mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
7406 							      &fib_work->fen_info);
7407 		}
7408 		fib_info_put(fib_work->fen_info.fi);
7409 		break;
7410 	case FIB_EVENT_ENTRY_DEL:
7411 		mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
7412 		fib_info_put(fib_work->fen_info.fi);
7413 		break;
7414 	case FIB_EVENT_NH_ADD:
7415 	case FIB_EVENT_NH_DEL:
7416 		mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
7417 					fib_work->fnh_info.fib_nh);
7418 		fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
7419 		break;
7420 	}
7421 	mutex_unlock(&mlxsw_sp->router->lock);
7422 	kfree(fib_work);
7423 }
7424 
7425 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
7426 {
7427 	struct mlxsw_sp_fib_event_work *fib_work =
7428 		    container_of(work, struct mlxsw_sp_fib_event_work, work);
7429 	struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work;
7430 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7431 	int err;
7432 
7433 	mutex_lock(&mlxsw_sp->router->lock);
7434 	mlxsw_sp_span_respin(mlxsw_sp);
7435 
7436 	switch (fib_work->event) {
7437 	case FIB_EVENT_ENTRY_REPLACE:
7438 		err = mlxsw_sp_router_fib6_replace(mlxsw_sp,
7439 						   fib6_work->rt_arr,
7440 						   fib6_work->nrt6);
7441 		if (err) {
7442 			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7443 			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7444 							      fib6_work->rt_arr,
7445 							      fib6_work->nrt6);
7446 		}
7447 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7448 		break;
7449 	case FIB_EVENT_ENTRY_APPEND:
7450 		err = mlxsw_sp_router_fib6_append(mlxsw_sp,
7451 						  fib6_work->rt_arr,
7452 						  fib6_work->nrt6);
7453 		if (err) {
7454 			dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n");
7455 			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7456 							      fib6_work->rt_arr,
7457 							      fib6_work->nrt6);
7458 		}
7459 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7460 		break;
7461 	case FIB_EVENT_ENTRY_DEL:
7462 		mlxsw_sp_router_fib6_del(mlxsw_sp,
7463 					 fib6_work->rt_arr,
7464 					 fib6_work->nrt6);
7465 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7466 		break;
7467 	}
7468 	mutex_unlock(&mlxsw_sp->router->lock);
7469 	kfree(fib_work);
7470 }
7471 
7472 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
7473 {
7474 	struct mlxsw_sp_fib_event_work *fib_work =
7475 		container_of(work, struct mlxsw_sp_fib_event_work, work);
7476 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7477 	bool replace;
7478 	int err;
7479 
7480 	rtnl_lock();
7481 	mutex_lock(&mlxsw_sp->router->lock);
7482 	switch (fib_work->event) {
7483 	case FIB_EVENT_ENTRY_REPLACE:
7484 	case FIB_EVENT_ENTRY_ADD:
7485 		replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
7486 
7487 		err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info,
7488 						replace);
7489 		if (err)
7490 			dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n");
7491 		mr_cache_put(fib_work->men_info.mfc);
7492 		break;
7493 	case FIB_EVENT_ENTRY_DEL:
7494 		mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info);
7495 		mr_cache_put(fib_work->men_info.mfc);
7496 		break;
7497 	case FIB_EVENT_VIF_ADD:
7498 		err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
7499 						    &fib_work->ven_info);
7500 		if (err)
7501 			dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n");
7502 		dev_put(fib_work->ven_info.dev);
7503 		break;
7504 	case FIB_EVENT_VIF_DEL:
7505 		mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
7506 					      &fib_work->ven_info);
7507 		dev_put(fib_work->ven_info.dev);
7508 		break;
7509 	}
7510 	mutex_unlock(&mlxsw_sp->router->lock);
7511 	rtnl_unlock();
7512 	kfree(fib_work);
7513 }
7514 
7515 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
7516 				       struct fib_notifier_info *info)
7517 {
7518 	struct fib_entry_notifier_info *fen_info;
7519 	struct fib_nh_notifier_info *fnh_info;
7520 
7521 	switch (fib_work->event) {
7522 	case FIB_EVENT_ENTRY_REPLACE:
7523 	case FIB_EVENT_ENTRY_DEL:
7524 		fen_info = container_of(info, struct fib_entry_notifier_info,
7525 					info);
7526 		fib_work->fen_info = *fen_info;
7527 		/* Take reference on fib_info to prevent it from being
7528 		 * freed while work is queued. Release it afterwards.
7529 		 */
7530 		fib_info_hold(fib_work->fen_info.fi);
7531 		break;
7532 	case FIB_EVENT_NH_ADD:
7533 	case FIB_EVENT_NH_DEL:
7534 		fnh_info = container_of(info, struct fib_nh_notifier_info,
7535 					info);
7536 		fib_work->fnh_info = *fnh_info;
7537 		fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
7538 		break;
7539 	}
7540 }
7541 
7542 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
7543 				      struct fib_notifier_info *info)
7544 {
7545 	struct fib6_entry_notifier_info *fen6_info;
7546 	int err;
7547 
7548 	switch (fib_work->event) {
7549 	case FIB_EVENT_ENTRY_REPLACE:
7550 	case FIB_EVENT_ENTRY_APPEND:
7551 	case FIB_EVENT_ENTRY_DEL:
7552 		fen6_info = container_of(info, struct fib6_entry_notifier_info,
7553 					 info);
7554 		err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work,
7555 						     fen6_info);
7556 		if (err)
7557 			return err;
7558 		break;
7559 	}
7560 
7561 	return 0;
7562 }
7563 
7564 static void
7565 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
7566 			    struct fib_notifier_info *info)
7567 {
7568 	switch (fib_work->event) {
7569 	case FIB_EVENT_ENTRY_REPLACE:
7570 	case FIB_EVENT_ENTRY_ADD:
7571 	case FIB_EVENT_ENTRY_DEL:
7572 		memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
7573 		mr_cache_hold(fib_work->men_info.mfc);
7574 		break;
7575 	case FIB_EVENT_VIF_ADD:
7576 	case FIB_EVENT_VIF_DEL:
7577 		memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
7578 		dev_hold(fib_work->ven_info.dev);
7579 		break;
7580 	}
7581 }
7582 
7583 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
7584 					  struct fib_notifier_info *info,
7585 					  struct mlxsw_sp *mlxsw_sp)
7586 {
7587 	struct netlink_ext_ack *extack = info->extack;
7588 	struct fib_rule_notifier_info *fr_info;
7589 	struct fib_rule *rule;
7590 	int err = 0;
7591 
7592 	/* nothing to do at the moment */
7593 	if (event == FIB_EVENT_RULE_DEL)
7594 		return 0;
7595 
7596 	fr_info = container_of(info, struct fib_rule_notifier_info, info);
7597 	rule = fr_info->rule;
7598 
7599 	/* Rule only affects locally generated traffic */
7600 	if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
7601 		return 0;
7602 
7603 	switch (info->family) {
7604 	case AF_INET:
7605 		if (!fib4_rule_default(rule) && !rule->l3mdev)
7606 			err = -EOPNOTSUPP;
7607 		break;
7608 	case AF_INET6:
7609 		if (!fib6_rule_default(rule) && !rule->l3mdev)
7610 			err = -EOPNOTSUPP;
7611 		break;
7612 	case RTNL_FAMILY_IPMR:
7613 		if (!ipmr_rule_default(rule) && !rule->l3mdev)
7614 			err = -EOPNOTSUPP;
7615 		break;
7616 	case RTNL_FAMILY_IP6MR:
7617 		if (!ip6mr_rule_default(rule) && !rule->l3mdev)
7618 			err = -EOPNOTSUPP;
7619 		break;
7620 	}
7621 
7622 	if (err < 0)
7623 		NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
7624 
7625 	return err;
7626 }
7627 
7628 /* Called with rcu_read_lock() */
7629 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
7630 				     unsigned long event, void *ptr)
7631 {
7632 	struct mlxsw_sp_fib_event_work *fib_work;
7633 	struct fib_notifier_info *info = ptr;
7634 	struct mlxsw_sp_router *router;
7635 	int err;
7636 
7637 	if ((info->family != AF_INET && info->family != AF_INET6 &&
7638 	     info->family != RTNL_FAMILY_IPMR &&
7639 	     info->family != RTNL_FAMILY_IP6MR))
7640 		return NOTIFY_DONE;
7641 
7642 	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7643 
7644 	switch (event) {
7645 	case FIB_EVENT_RULE_ADD:
7646 	case FIB_EVENT_RULE_DEL:
7647 		err = mlxsw_sp_router_fib_rule_event(event, info,
7648 						     router->mlxsw_sp);
7649 		return notifier_from_errno(err);
7650 	case FIB_EVENT_ENTRY_ADD:
7651 	case FIB_EVENT_ENTRY_REPLACE:
7652 	case FIB_EVENT_ENTRY_APPEND:
7653 		if (info->family == AF_INET) {
7654 			struct fib_entry_notifier_info *fen_info = ptr;
7655 
7656 			if (fen_info->fi->fib_nh_is_v6) {
7657 				NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
7658 				return notifier_from_errno(-EINVAL);
7659 			}
7660 		}
7661 		break;
7662 	}
7663 
7664 	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
7665 	if (!fib_work)
7666 		return NOTIFY_BAD;
7667 
7668 	fib_work->mlxsw_sp = router->mlxsw_sp;
7669 	fib_work->event = event;
7670 
7671 	switch (info->family) {
7672 	case AF_INET:
7673 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
7674 		mlxsw_sp_router_fib4_event(fib_work, info);
7675 		break;
7676 	case AF_INET6:
7677 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
7678 		err = mlxsw_sp_router_fib6_event(fib_work, info);
7679 		if (err)
7680 			goto err_fib_event;
7681 		break;
7682 	case RTNL_FAMILY_IP6MR:
7683 	case RTNL_FAMILY_IPMR:
7684 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
7685 		mlxsw_sp_router_fibmr_event(fib_work, info);
7686 		break;
7687 	}
7688 
7689 	mlxsw_core_schedule_work(&fib_work->work);
7690 
7691 	return NOTIFY_DONE;
7692 
7693 err_fib_event:
7694 	kfree(fib_work);
7695 	return NOTIFY_BAD;
7696 }
7697 
7698 static struct mlxsw_sp_rif *
7699 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
7700 			 const struct net_device *dev)
7701 {
7702 	int i;
7703 
7704 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
7705 		if (mlxsw_sp->router->rifs[i] &&
7706 		    mlxsw_sp->router->rifs[i]->dev == dev)
7707 			return mlxsw_sp->router->rifs[i];
7708 
7709 	return NULL;
7710 }
7711 
7712 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
7713 			 const struct net_device *dev)
7714 {
7715 	struct mlxsw_sp_rif *rif;
7716 
7717 	mutex_lock(&mlxsw_sp->router->lock);
7718 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7719 	mutex_unlock(&mlxsw_sp->router->lock);
7720 
7721 	return rif;
7722 }
7723 
7724 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
7725 {
7726 	struct mlxsw_sp_rif *rif;
7727 	u16 vid = 0;
7728 
7729 	mutex_lock(&mlxsw_sp->router->lock);
7730 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7731 	if (!rif)
7732 		goto out;
7733 
7734 	/* We only return the VID for VLAN RIFs. Otherwise we return an
7735 	 * invalid value (0).
7736 	 */
7737 	if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
7738 		goto out;
7739 
7740 	vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7741 
7742 out:
7743 	mutex_unlock(&mlxsw_sp->router->lock);
7744 	return vid;
7745 }
7746 
7747 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
7748 {
7749 	char ritr_pl[MLXSW_REG_RITR_LEN];
7750 	int err;
7751 
7752 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
7753 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7754 	if (err)
7755 		return err;
7756 
7757 	mlxsw_reg_ritr_enable_set(ritr_pl, false);
7758 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7759 }
7760 
7761 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
7762 					  struct mlxsw_sp_rif *rif)
7763 {
7764 	mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
7765 	mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
7766 	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
7767 }
7768 
7769 static bool
7770 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
7771 			   unsigned long event)
7772 {
7773 	struct inet6_dev *inet6_dev;
7774 	bool addr_list_empty = true;
7775 	struct in_device *idev;
7776 
7777 	switch (event) {
7778 	case NETDEV_UP:
7779 		return rif == NULL;
7780 	case NETDEV_DOWN:
7781 		rcu_read_lock();
7782 		idev = __in_dev_get_rcu(dev);
7783 		if (idev && idev->ifa_list)
7784 			addr_list_empty = false;
7785 
7786 		inet6_dev = __in6_dev_get(dev);
7787 		if (addr_list_empty && inet6_dev &&
7788 		    !list_empty(&inet6_dev->addr_list))
7789 			addr_list_empty = false;
7790 		rcu_read_unlock();
7791 
7792 		/* macvlans do not have a RIF, but rather piggy back on the
7793 		 * RIF of their lower device.
7794 		 */
7795 		if (netif_is_macvlan(dev) && addr_list_empty)
7796 			return true;
7797 
7798 		if (rif && addr_list_empty &&
7799 		    !netif_is_l3_slave(rif->dev))
7800 			return true;
7801 		/* It is possible we already removed the RIF ourselves
7802 		 * if it was assigned to a netdev that is now a bridge
7803 		 * or LAG slave.
7804 		 */
7805 		return false;
7806 	}
7807 
7808 	return false;
7809 }
7810 
7811 static enum mlxsw_sp_rif_type
7812 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
7813 		      const struct net_device *dev)
7814 {
7815 	enum mlxsw_sp_fid_type type;
7816 
7817 	if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
7818 		return MLXSW_SP_RIF_TYPE_IPIP_LB;
7819 
7820 	/* Otherwise RIF type is derived from the type of the underlying FID. */
7821 	if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
7822 		type = MLXSW_SP_FID_TYPE_8021Q;
7823 	else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
7824 		type = MLXSW_SP_FID_TYPE_8021Q;
7825 	else if (netif_is_bridge_master(dev))
7826 		type = MLXSW_SP_FID_TYPE_8021D;
7827 	else
7828 		type = MLXSW_SP_FID_TYPE_RFID;
7829 
7830 	return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
7831 }
7832 
7833 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index,
7834 				    u8 rif_entries)
7835 {
7836 	*p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table,
7837 				      rif_entries);
7838 	if (*p_rif_index == 0)
7839 		return -ENOBUFS;
7840 	*p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET;
7841 
7842 	/* RIF indexes must be aligned to the allocation size. */
7843 	WARN_ON_ONCE(*p_rif_index % rif_entries);
7844 
7845 	return 0;
7846 }
7847 
7848 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
7849 				    u8 rif_entries)
7850 {
7851 	gen_pool_free(mlxsw_sp->router->rifs_table,
7852 		      MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries);
7853 }
7854 
7855 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
7856 					       u16 vr_id,
7857 					       struct net_device *l3_dev)
7858 {
7859 	struct mlxsw_sp_rif *rif;
7860 
7861 	rif = kzalloc(rif_size, GFP_KERNEL);
7862 	if (!rif)
7863 		return NULL;
7864 
7865 	INIT_LIST_HEAD(&rif->nexthop_list);
7866 	INIT_LIST_HEAD(&rif->neigh_list);
7867 	if (l3_dev) {
7868 		ether_addr_copy(rif->addr, l3_dev->dev_addr);
7869 		rif->mtu = l3_dev->mtu;
7870 		rif->dev = l3_dev;
7871 	}
7872 	rif->vr_id = vr_id;
7873 	rif->rif_index = rif_index;
7874 
7875 	return rif;
7876 }
7877 
7878 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
7879 					   u16 rif_index)
7880 {
7881 	return mlxsw_sp->router->rifs[rif_index];
7882 }
7883 
7884 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
7885 {
7886 	return rif->rif_index;
7887 }
7888 
7889 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7890 {
7891 	return lb_rif->common.rif_index;
7892 }
7893 
7894 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7895 {
7896 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
7897 	struct mlxsw_sp_vr *ul_vr;
7898 
7899 	ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
7900 	if (WARN_ON(IS_ERR(ul_vr)))
7901 		return 0;
7902 
7903 	return ul_vr->id;
7904 }
7905 
7906 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
7907 {
7908 	return lb_rif->ul_rif_id;
7909 }
7910 
7911 static bool
7912 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif)
7913 {
7914 	return mlxsw_sp_rif_counter_valid_get(rif,
7915 					      MLXSW_SP_RIF_COUNTER_EGRESS) &&
7916 	       mlxsw_sp_rif_counter_valid_get(rif,
7917 					      MLXSW_SP_RIF_COUNTER_INGRESS);
7918 }
7919 
7920 static int
7921 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif)
7922 {
7923 	int err;
7924 
7925 	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
7926 	if (err)
7927 		return err;
7928 
7929 	/* Clear stale data. */
7930 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
7931 					       MLXSW_SP_RIF_COUNTER_INGRESS,
7932 					       NULL);
7933 	if (err)
7934 		goto err_clear_ingress;
7935 
7936 	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
7937 	if (err)
7938 		goto err_alloc_egress;
7939 
7940 	/* Clear stale data. */
7941 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
7942 					       MLXSW_SP_RIF_COUNTER_EGRESS,
7943 					       NULL);
7944 	if (err)
7945 		goto err_clear_egress;
7946 
7947 	return 0;
7948 
7949 err_clear_egress:
7950 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
7951 err_alloc_egress:
7952 err_clear_ingress:
7953 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
7954 	return err;
7955 }
7956 
7957 static void
7958 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif)
7959 {
7960 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
7961 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
7962 }
7963 
7964 static void
7965 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif,
7966 					  struct netdev_notifier_offload_xstats_info *info)
7967 {
7968 	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
7969 		return;
7970 	netdev_offload_xstats_report_used(info->report_used);
7971 }
7972 
7973 static int
7974 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif,
7975 				    struct rtnl_hw_stats64 *p_stats)
7976 {
7977 	struct mlxsw_sp_rif_counter_set_basic ingress;
7978 	struct mlxsw_sp_rif_counter_set_basic egress;
7979 	int err;
7980 
7981 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
7982 					       MLXSW_SP_RIF_COUNTER_INGRESS,
7983 					       &ingress);
7984 	if (err)
7985 		return err;
7986 
7987 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
7988 					       MLXSW_SP_RIF_COUNTER_EGRESS,
7989 					       &egress);
7990 	if (err)
7991 		return err;
7992 
7993 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX)		\
7994 		((SET.good_unicast_ ## SFX) +		\
7995 		 (SET.good_multicast_ ## SFX) +		\
7996 		 (SET.good_broadcast_ ## SFX))
7997 
7998 	p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets);
7999 	p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets);
8000 	p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes);
8001 	p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes);
8002 	p_stats->rx_errors = ingress.error_packets;
8003 	p_stats->tx_errors = egress.error_packets;
8004 	p_stats->rx_dropped = ingress.discard_packets;
8005 	p_stats->tx_dropped = egress.discard_packets;
8006 	p_stats->multicast = ingress.good_multicast_packets +
8007 			     ingress.good_broadcast_packets;
8008 
8009 #undef MLXSW_SP_ROUTER_ALL_GOOD
8010 
8011 	return 0;
8012 }
8013 
8014 static int
8015 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif,
8016 					   struct netdev_notifier_offload_xstats_info *info)
8017 {
8018 	struct rtnl_hw_stats64 stats = {};
8019 	int err;
8020 
8021 	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8022 		return 0;
8023 
8024 	err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats);
8025 	if (err)
8026 		return err;
8027 
8028 	netdev_offload_xstats_report_delta(info->report_delta, &stats);
8029 	return 0;
8030 }
8031 
8032 struct mlxsw_sp_router_hwstats_notify_work {
8033 	struct work_struct work;
8034 	struct net_device *dev;
8035 };
8036 
8037 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work)
8038 {
8039 	struct mlxsw_sp_router_hwstats_notify_work *hws_work =
8040 		container_of(work, struct mlxsw_sp_router_hwstats_notify_work,
8041 			     work);
8042 
8043 	rtnl_lock();
8044 	rtnl_offload_xstats_notify(hws_work->dev);
8045 	rtnl_unlock();
8046 	dev_put(hws_work->dev);
8047 	kfree(hws_work);
8048 }
8049 
8050 static void
8051 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev)
8052 {
8053 	struct mlxsw_sp_router_hwstats_notify_work *hws_work;
8054 
8055 	/* To collect notification payload, the core ends up sending another
8056 	 * notifier block message, which would deadlock on the attempt to
8057 	 * acquire the router lock again. Just postpone the notification until
8058 	 * later.
8059 	 */
8060 
8061 	hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL);
8062 	if (!hws_work)
8063 		return;
8064 
8065 	INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work);
8066 	dev_hold(dev);
8067 	hws_work->dev = dev;
8068 	mlxsw_core_schedule_work(&hws_work->work);
8069 }
8070 
8071 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
8072 {
8073 	return rif->dev->ifindex;
8074 }
8075 
8076 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
8077 {
8078 	return rif->dev;
8079 }
8080 
8081 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif)
8082 {
8083 	struct rtnl_hw_stats64 stats = {};
8084 
8085 	if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats))
8086 		netdev_offload_xstats_push_delta(rif->dev,
8087 						 NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8088 						 &stats);
8089 }
8090 
8091 static struct mlxsw_sp_rif *
8092 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
8093 		    const struct mlxsw_sp_rif_params *params,
8094 		    struct netlink_ext_ack *extack)
8095 {
8096 	u8 rif_entries = params->double_entry ? 2 : 1;
8097 	u32 tb_id = l3mdev_fib_table(params->dev);
8098 	const struct mlxsw_sp_rif_ops *ops;
8099 	struct mlxsw_sp_fid *fid = NULL;
8100 	enum mlxsw_sp_rif_type type;
8101 	struct mlxsw_sp_rif *rif;
8102 	struct mlxsw_sp_vr *vr;
8103 	u16 rif_index;
8104 	int i, err;
8105 
8106 	type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
8107 	ops = mlxsw_sp->router->rif_ops_arr[type];
8108 
8109 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
8110 	if (IS_ERR(vr))
8111 		return ERR_CAST(vr);
8112 	vr->rif_count++;
8113 
8114 	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
8115 	if (err) {
8116 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8117 		goto err_rif_index_alloc;
8118 	}
8119 
8120 	rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
8121 	if (!rif) {
8122 		err = -ENOMEM;
8123 		goto err_rif_alloc;
8124 	}
8125 	dev_hold(rif->dev);
8126 	mlxsw_sp->router->rifs[rif_index] = rif;
8127 	rif->mlxsw_sp = mlxsw_sp;
8128 	rif->ops = ops;
8129 	rif->rif_entries = rif_entries;
8130 
8131 	if (ops->fid_get) {
8132 		fid = ops->fid_get(rif, extack);
8133 		if (IS_ERR(fid)) {
8134 			err = PTR_ERR(fid);
8135 			goto err_fid_get;
8136 		}
8137 		rif->fid = fid;
8138 	}
8139 
8140 	if (ops->setup)
8141 		ops->setup(rif, params);
8142 
8143 	err = ops->configure(rif, extack);
8144 	if (err)
8145 		goto err_configure;
8146 
8147 	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8148 		err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
8149 		if (err)
8150 			goto err_mr_rif_add;
8151 	}
8152 
8153 	if (netdev_offload_xstats_enabled(rif->dev,
8154 					  NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8155 		err = mlxsw_sp_router_port_l3_stats_enable(rif);
8156 		if (err)
8157 			goto err_stats_enable;
8158 		mlxsw_sp_router_hwstats_notify_schedule(rif->dev);
8159 	} else {
8160 		mlxsw_sp_rif_counters_alloc(rif);
8161 	}
8162 
8163 	atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
8164 	return rif;
8165 
8166 err_stats_enable:
8167 err_mr_rif_add:
8168 	for (i--; i >= 0; i--)
8169 		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8170 	ops->deconfigure(rif);
8171 err_configure:
8172 	if (fid)
8173 		mlxsw_sp_fid_put(fid);
8174 err_fid_get:
8175 	mlxsw_sp->router->rifs[rif_index] = NULL;
8176 	dev_put(rif->dev);
8177 	kfree(rif);
8178 err_rif_alloc:
8179 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8180 err_rif_index_alloc:
8181 	vr->rif_count--;
8182 	mlxsw_sp_vr_put(mlxsw_sp, vr);
8183 	return ERR_PTR(err);
8184 }
8185 
8186 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
8187 {
8188 	const struct mlxsw_sp_rif_ops *ops = rif->ops;
8189 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8190 	struct mlxsw_sp_fid *fid = rif->fid;
8191 	u8 rif_entries = rif->rif_entries;
8192 	u16 rif_index = rif->rif_index;
8193 	struct mlxsw_sp_vr *vr;
8194 	int i;
8195 
8196 	atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
8197 	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8198 	vr = &mlxsw_sp->router->vrs[rif->vr_id];
8199 
8200 	if (netdev_offload_xstats_enabled(rif->dev,
8201 					  NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8202 		mlxsw_sp_rif_push_l3_stats(rif);
8203 		mlxsw_sp_router_port_l3_stats_disable(rif);
8204 		mlxsw_sp_router_hwstats_notify_schedule(rif->dev);
8205 	} else {
8206 		mlxsw_sp_rif_counters_free(rif);
8207 	}
8208 
8209 	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
8210 		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8211 	ops->deconfigure(rif);
8212 	if (fid)
8213 		/* Loopback RIFs are not associated with a FID. */
8214 		mlxsw_sp_fid_put(fid);
8215 	mlxsw_sp->router->rifs[rif->rif_index] = NULL;
8216 	dev_put(rif->dev);
8217 	kfree(rif);
8218 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8219 	vr->rif_count--;
8220 	mlxsw_sp_vr_put(mlxsw_sp, vr);
8221 }
8222 
8223 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
8224 				 struct net_device *dev)
8225 {
8226 	struct mlxsw_sp_rif *rif;
8227 
8228 	mutex_lock(&mlxsw_sp->router->lock);
8229 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8230 	if (!rif)
8231 		goto out;
8232 	mlxsw_sp_rif_destroy(rif);
8233 out:
8234 	mutex_unlock(&mlxsw_sp->router->lock);
8235 }
8236 
8237 static void
8238 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
8239 				 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8240 {
8241 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8242 
8243 	params->vid = mlxsw_sp_port_vlan->vid;
8244 	params->lag = mlxsw_sp_port->lagged;
8245 	if (params->lag)
8246 		params->lag_id = mlxsw_sp_port->lag_id;
8247 	else
8248 		params->system_port = mlxsw_sp_port->local_port;
8249 }
8250 
8251 static struct mlxsw_sp_rif_subport *
8252 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
8253 {
8254 	return container_of(rif, struct mlxsw_sp_rif_subport, common);
8255 }
8256 
8257 static struct mlxsw_sp_rif *
8258 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
8259 			 const struct mlxsw_sp_rif_params *params,
8260 			 struct netlink_ext_ack *extack)
8261 {
8262 	struct mlxsw_sp_rif_subport *rif_subport;
8263 	struct mlxsw_sp_rif *rif;
8264 
8265 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
8266 	if (!rif)
8267 		return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
8268 
8269 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8270 	refcount_inc(&rif_subport->ref_count);
8271 	return rif;
8272 }
8273 
8274 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
8275 {
8276 	struct mlxsw_sp_rif_subport *rif_subport;
8277 
8278 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8279 	if (!refcount_dec_and_test(&rif_subport->ref_count))
8280 		return;
8281 
8282 	mlxsw_sp_rif_destroy(rif);
8283 }
8284 
8285 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp,
8286 						struct mlxsw_sp_rif_mac_profile *profile,
8287 						struct netlink_ext_ack *extack)
8288 {
8289 	u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
8290 	struct mlxsw_sp_router *router = mlxsw_sp->router;
8291 	int id;
8292 
8293 	id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0,
8294 		       max_rif_mac_profiles, GFP_KERNEL);
8295 
8296 	if (id >= 0) {
8297 		profile->id = id;
8298 		return 0;
8299 	}
8300 
8301 	if (id == -ENOSPC)
8302 		NL_SET_ERR_MSG_MOD(extack,
8303 				   "Exceeded number of supported router interface MAC profiles");
8304 
8305 	return id;
8306 }
8307 
8308 static struct mlxsw_sp_rif_mac_profile *
8309 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile)
8310 {
8311 	struct mlxsw_sp_rif_mac_profile *profile;
8312 
8313 	profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr,
8314 			     mac_profile);
8315 	WARN_ON(!profile);
8316 	return profile;
8317 }
8318 
8319 static struct mlxsw_sp_rif_mac_profile *
8320 mlxsw_sp_rif_mac_profile_alloc(const char *mac)
8321 {
8322 	struct mlxsw_sp_rif_mac_profile *profile;
8323 
8324 	profile = kzalloc(sizeof(*profile), GFP_KERNEL);
8325 	if (!profile)
8326 		return NULL;
8327 
8328 	ether_addr_copy(profile->mac_prefix, mac);
8329 	refcount_set(&profile->ref_count, 1);
8330 	return profile;
8331 }
8332 
8333 static struct mlxsw_sp_rif_mac_profile *
8334 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac)
8335 {
8336 	struct mlxsw_sp_router *router = mlxsw_sp->router;
8337 	struct mlxsw_sp_rif_mac_profile *profile;
8338 	int id;
8339 
8340 	idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) {
8341 		if (ether_addr_equal_masked(profile->mac_prefix, mac,
8342 					    mlxsw_sp->mac_mask))
8343 			return profile;
8344 	}
8345 
8346 	return NULL;
8347 }
8348 
8349 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv)
8350 {
8351 	const struct mlxsw_sp *mlxsw_sp = priv;
8352 
8353 	return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count);
8354 }
8355 
8356 static u64 mlxsw_sp_rifs_occ_get(void *priv)
8357 {
8358 	const struct mlxsw_sp *mlxsw_sp = priv;
8359 
8360 	return atomic_read(&mlxsw_sp->router->rifs_count);
8361 }
8362 
8363 static struct mlxsw_sp_rif_mac_profile *
8364 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac,
8365 				struct netlink_ext_ack *extack)
8366 {
8367 	struct mlxsw_sp_rif_mac_profile *profile;
8368 	int err;
8369 
8370 	profile = mlxsw_sp_rif_mac_profile_alloc(mac);
8371 	if (!profile)
8372 		return ERR_PTR(-ENOMEM);
8373 
8374 	err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack);
8375 	if (err)
8376 		goto profile_index_alloc_err;
8377 
8378 	atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count);
8379 	return profile;
8380 
8381 profile_index_alloc_err:
8382 	kfree(profile);
8383 	return ERR_PTR(err);
8384 }
8385 
8386 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp,
8387 					     u8 mac_profile)
8388 {
8389 	struct mlxsw_sp_rif_mac_profile *profile;
8390 
8391 	atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count);
8392 	profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile);
8393 	kfree(profile);
8394 }
8395 
8396 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp,
8397 					const char *mac, u8 *p_mac_profile,
8398 					struct netlink_ext_ack *extack)
8399 {
8400 	struct mlxsw_sp_rif_mac_profile *profile;
8401 
8402 	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac);
8403 	if (profile) {
8404 		refcount_inc(&profile->ref_count);
8405 		goto out;
8406 	}
8407 
8408 	profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack);
8409 	if (IS_ERR(profile))
8410 		return PTR_ERR(profile);
8411 
8412 out:
8413 	*p_mac_profile = profile->id;
8414 	return 0;
8415 }
8416 
8417 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp,
8418 					 u8 mac_profile)
8419 {
8420 	struct mlxsw_sp_rif_mac_profile *profile;
8421 
8422 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8423 			   mac_profile);
8424 	if (WARN_ON(!profile))
8425 		return;
8426 
8427 	if (!refcount_dec_and_test(&profile->ref_count))
8428 		return;
8429 
8430 	mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile);
8431 }
8432 
8433 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif)
8434 {
8435 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8436 	struct mlxsw_sp_rif_mac_profile *profile;
8437 
8438 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8439 			   rif->mac_profile_id);
8440 	if (WARN_ON(!profile))
8441 		return false;
8442 
8443 	return refcount_read(&profile->ref_count) > 1;
8444 }
8445 
8446 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif,
8447 					 const char *new_mac)
8448 {
8449 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8450 	struct mlxsw_sp_rif_mac_profile *profile;
8451 
8452 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8453 			   rif->mac_profile_id);
8454 	if (WARN_ON(!profile))
8455 		return -EINVAL;
8456 
8457 	ether_addr_copy(profile->mac_prefix, new_mac);
8458 	return 0;
8459 }
8460 
8461 static int
8462 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp,
8463 				 struct mlxsw_sp_rif *rif,
8464 				 const char *new_mac,
8465 				 struct netlink_ext_ack *extack)
8466 {
8467 	u8 mac_profile;
8468 	int err;
8469 
8470 	if (!mlxsw_sp_rif_mac_profile_is_shared(rif) &&
8471 	    !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac))
8472 		return mlxsw_sp_rif_mac_profile_edit(rif, new_mac);
8473 
8474 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac,
8475 					   &mac_profile, extack);
8476 	if (err)
8477 		return err;
8478 
8479 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id);
8480 	rif->mac_profile_id = mac_profile;
8481 	return 0;
8482 }
8483 
8484 static int
8485 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8486 				 struct net_device *l3_dev,
8487 				 struct netlink_ext_ack *extack)
8488 {
8489 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8490 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
8491 	struct mlxsw_sp_rif_params params = {
8492 		.dev = l3_dev,
8493 	};
8494 	u16 vid = mlxsw_sp_port_vlan->vid;
8495 	struct mlxsw_sp_rif *rif;
8496 	struct mlxsw_sp_fid *fid;
8497 	int err;
8498 
8499 	mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
8500 	rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
8501 	if (IS_ERR(rif))
8502 		return PTR_ERR(rif);
8503 
8504 	/* FID was already created, just take a reference */
8505 	fid = rif->ops->fid_get(rif, extack);
8506 	err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
8507 	if (err)
8508 		goto err_fid_port_vid_map;
8509 
8510 	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
8511 	if (err)
8512 		goto err_port_vid_learning_set;
8513 
8514 	err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
8515 					BR_STATE_FORWARDING);
8516 	if (err)
8517 		goto err_port_vid_stp_set;
8518 
8519 	mlxsw_sp_port_vlan->fid = fid;
8520 
8521 	return 0;
8522 
8523 err_port_vid_stp_set:
8524 	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8525 err_port_vid_learning_set:
8526 	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8527 err_fid_port_vid_map:
8528 	mlxsw_sp_fid_put(fid);
8529 	mlxsw_sp_rif_subport_put(rif);
8530 	return err;
8531 }
8532 
8533 static void
8534 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8535 {
8536 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8537 	struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
8538 	struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
8539 	u16 vid = mlxsw_sp_port_vlan->vid;
8540 
8541 	if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
8542 		return;
8543 
8544 	mlxsw_sp_port_vlan->fid = NULL;
8545 	mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
8546 	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8547 	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8548 	mlxsw_sp_fid_put(fid);
8549 	mlxsw_sp_rif_subport_put(rif);
8550 }
8551 
8552 int
8553 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8554 			       struct net_device *l3_dev,
8555 			       struct netlink_ext_ack *extack)
8556 {
8557 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8558 	struct mlxsw_sp_rif *rif;
8559 	int err = 0;
8560 
8561 	mutex_lock(&mlxsw_sp->router->lock);
8562 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
8563 	if (!rif)
8564 		goto out;
8565 
8566 	err = __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
8567 					       extack);
8568 out:
8569 	mutex_unlock(&mlxsw_sp->router->lock);
8570 	return err;
8571 }
8572 
8573 void
8574 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8575 {
8576 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8577 
8578 	mutex_lock(&mlxsw_sp->router->lock);
8579 	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
8580 	mutex_unlock(&mlxsw_sp->router->lock);
8581 }
8582 
8583 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
8584 					     struct net_device *port_dev,
8585 					     unsigned long event, u16 vid,
8586 					     struct netlink_ext_ack *extack)
8587 {
8588 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
8589 	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
8590 
8591 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
8592 	if (WARN_ON(!mlxsw_sp_port_vlan))
8593 		return -EINVAL;
8594 
8595 	switch (event) {
8596 	case NETDEV_UP:
8597 		return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
8598 							l3_dev, extack);
8599 	case NETDEV_DOWN:
8600 		__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
8601 		break;
8602 	}
8603 
8604 	return 0;
8605 }
8606 
8607 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
8608 					unsigned long event,
8609 					struct netlink_ext_ack *extack)
8610 {
8611 	if (netif_is_any_bridge_port(port_dev) || netif_is_lag_port(port_dev))
8612 		return 0;
8613 
8614 	return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
8615 						 MLXSW_SP_DEFAULT_VID, extack);
8616 }
8617 
8618 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
8619 					 struct net_device *lag_dev,
8620 					 unsigned long event, u16 vid,
8621 					 struct netlink_ext_ack *extack)
8622 {
8623 	struct net_device *port_dev;
8624 	struct list_head *iter;
8625 	int err;
8626 
8627 	netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
8628 		if (mlxsw_sp_port_dev_check(port_dev)) {
8629 			err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
8630 								port_dev,
8631 								event, vid,
8632 								extack);
8633 			if (err)
8634 				return err;
8635 		}
8636 	}
8637 
8638 	return 0;
8639 }
8640 
8641 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
8642 				       unsigned long event,
8643 				       struct netlink_ext_ack *extack)
8644 {
8645 	if (netif_is_bridge_port(lag_dev))
8646 		return 0;
8647 
8648 	return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
8649 					     MLXSW_SP_DEFAULT_VID, extack);
8650 }
8651 
8652 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
8653 					  struct net_device *l3_dev,
8654 					  unsigned long event,
8655 					  struct netlink_ext_ack *extack)
8656 {
8657 	struct mlxsw_sp_rif_params params = {
8658 		.dev = l3_dev,
8659 	};
8660 	struct mlxsw_sp_rif *rif;
8661 
8662 	switch (event) {
8663 	case NETDEV_UP:
8664 		if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) {
8665 			u16 proto;
8666 
8667 			br_vlan_get_proto(l3_dev, &proto);
8668 			if (proto == ETH_P_8021AD) {
8669 				NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
8670 				return -EOPNOTSUPP;
8671 			}
8672 		}
8673 		rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
8674 		if (IS_ERR(rif))
8675 			return PTR_ERR(rif);
8676 		break;
8677 	case NETDEV_DOWN:
8678 		rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
8679 		mlxsw_sp_rif_destroy(rif);
8680 		break;
8681 	}
8682 
8683 	return 0;
8684 }
8685 
8686 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
8687 					struct net_device *vlan_dev,
8688 					unsigned long event,
8689 					struct netlink_ext_ack *extack)
8690 {
8691 	struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
8692 	u16 vid = vlan_dev_vlan_id(vlan_dev);
8693 
8694 	if (netif_is_bridge_port(vlan_dev))
8695 		return 0;
8696 
8697 	if (mlxsw_sp_port_dev_check(real_dev))
8698 		return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
8699 							 event, vid, extack);
8700 	else if (netif_is_lag_master(real_dev))
8701 		return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
8702 						     vid, extack);
8703 	else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
8704 		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
8705 						      extack);
8706 
8707 	return 0;
8708 }
8709 
8710 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
8711 {
8712 	u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
8713 	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
8714 
8715 	return ether_addr_equal_masked(mac, vrrp4, mask);
8716 }
8717 
8718 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
8719 {
8720 	u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
8721 	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
8722 
8723 	return ether_addr_equal_masked(mac, vrrp6, mask);
8724 }
8725 
8726 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8727 				const u8 *mac, bool adding)
8728 {
8729 	char ritr_pl[MLXSW_REG_RITR_LEN];
8730 	u8 vrrp_id = adding ? mac[5] : 0;
8731 	int err;
8732 
8733 	if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
8734 	    !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
8735 		return 0;
8736 
8737 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
8738 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8739 	if (err)
8740 		return err;
8741 
8742 	if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
8743 		mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
8744 	else
8745 		mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
8746 
8747 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8748 }
8749 
8750 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
8751 				    const struct net_device *macvlan_dev,
8752 				    struct netlink_ext_ack *extack)
8753 {
8754 	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
8755 	struct mlxsw_sp_rif *rif;
8756 	int err;
8757 
8758 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
8759 	if (!rif) {
8760 		NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
8761 		return -EOPNOTSUPP;
8762 	}
8763 
8764 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8765 				  mlxsw_sp_fid_index(rif->fid), true);
8766 	if (err)
8767 		return err;
8768 
8769 	err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
8770 				   macvlan_dev->dev_addr, true);
8771 	if (err)
8772 		goto err_rif_vrrp_add;
8773 
8774 	/* Make sure the bridge driver does not have this MAC pointing at
8775 	 * some other port.
8776 	 */
8777 	if (rif->ops->fdb_del)
8778 		rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
8779 
8780 	return 0;
8781 
8782 err_rif_vrrp_add:
8783 	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8784 			    mlxsw_sp_fid_index(rif->fid), false);
8785 	return err;
8786 }
8787 
8788 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
8789 				       const struct net_device *macvlan_dev)
8790 {
8791 	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
8792 	struct mlxsw_sp_rif *rif;
8793 
8794 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
8795 	/* If we do not have a RIF, then we already took care of
8796 	 * removing the macvlan's MAC during RIF deletion.
8797 	 */
8798 	if (!rif)
8799 		return;
8800 	mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
8801 			     false);
8802 	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
8803 			    mlxsw_sp_fid_index(rif->fid), false);
8804 }
8805 
8806 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
8807 			      const struct net_device *macvlan_dev)
8808 {
8809 	mutex_lock(&mlxsw_sp->router->lock);
8810 	__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
8811 	mutex_unlock(&mlxsw_sp->router->lock);
8812 }
8813 
8814 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
8815 					   struct net_device *macvlan_dev,
8816 					   unsigned long event,
8817 					   struct netlink_ext_ack *extack)
8818 {
8819 	switch (event) {
8820 	case NETDEV_UP:
8821 		return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
8822 	case NETDEV_DOWN:
8823 		__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
8824 		break;
8825 	}
8826 
8827 	return 0;
8828 }
8829 
8830 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
8831 				     struct net_device *dev,
8832 				     unsigned long event,
8833 				     struct netlink_ext_ack *extack)
8834 {
8835 	if (mlxsw_sp_port_dev_check(dev))
8836 		return mlxsw_sp_inetaddr_port_event(dev, event, extack);
8837 	else if (netif_is_lag_master(dev))
8838 		return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
8839 	else if (netif_is_bridge_master(dev))
8840 		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
8841 						      extack);
8842 	else if (is_vlan_dev(dev))
8843 		return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
8844 						    extack);
8845 	else if (netif_is_macvlan(dev))
8846 		return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
8847 						       extack);
8848 	else
8849 		return 0;
8850 }
8851 
8852 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
8853 				   unsigned long event, void *ptr)
8854 {
8855 	struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
8856 	struct net_device *dev = ifa->ifa_dev->dev;
8857 	struct mlxsw_sp_router *router;
8858 	struct mlxsw_sp_rif *rif;
8859 	int err = 0;
8860 
8861 	/* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
8862 	if (event == NETDEV_UP)
8863 		return NOTIFY_DONE;
8864 
8865 	router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
8866 	mutex_lock(&router->lock);
8867 	rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
8868 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
8869 		goto out;
8870 
8871 	err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
8872 out:
8873 	mutex_unlock(&router->lock);
8874 	return notifier_from_errno(err);
8875 }
8876 
8877 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
8878 				  unsigned long event, void *ptr)
8879 {
8880 	struct in_validator_info *ivi = (struct in_validator_info *) ptr;
8881 	struct net_device *dev = ivi->ivi_dev->dev;
8882 	struct mlxsw_sp *mlxsw_sp;
8883 	struct mlxsw_sp_rif *rif;
8884 	int err = 0;
8885 
8886 	mlxsw_sp = mlxsw_sp_lower_get(dev);
8887 	if (!mlxsw_sp)
8888 		return NOTIFY_DONE;
8889 
8890 	mutex_lock(&mlxsw_sp->router->lock);
8891 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8892 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
8893 		goto out;
8894 
8895 	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
8896 out:
8897 	mutex_unlock(&mlxsw_sp->router->lock);
8898 	return notifier_from_errno(err);
8899 }
8900 
8901 struct mlxsw_sp_inet6addr_event_work {
8902 	struct work_struct work;
8903 	struct mlxsw_sp *mlxsw_sp;
8904 	struct net_device *dev;
8905 	unsigned long event;
8906 };
8907 
8908 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
8909 {
8910 	struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
8911 		container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
8912 	struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
8913 	struct net_device *dev = inet6addr_work->dev;
8914 	unsigned long event = inet6addr_work->event;
8915 	struct mlxsw_sp_rif *rif;
8916 
8917 	rtnl_lock();
8918 	mutex_lock(&mlxsw_sp->router->lock);
8919 
8920 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8921 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
8922 		goto out;
8923 
8924 	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
8925 out:
8926 	mutex_unlock(&mlxsw_sp->router->lock);
8927 	rtnl_unlock();
8928 	dev_put(dev);
8929 	kfree(inet6addr_work);
8930 }
8931 
8932 /* Called with rcu_read_lock() */
8933 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
8934 				    unsigned long event, void *ptr)
8935 {
8936 	struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
8937 	struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
8938 	struct net_device *dev = if6->idev->dev;
8939 	struct mlxsw_sp_router *router;
8940 
8941 	/* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
8942 	if (event == NETDEV_UP)
8943 		return NOTIFY_DONE;
8944 
8945 	inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
8946 	if (!inet6addr_work)
8947 		return NOTIFY_BAD;
8948 
8949 	router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
8950 	INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
8951 	inet6addr_work->mlxsw_sp = router->mlxsw_sp;
8952 	inet6addr_work->dev = dev;
8953 	inet6addr_work->event = event;
8954 	dev_hold(dev);
8955 	mlxsw_core_schedule_work(&inet6addr_work->work);
8956 
8957 	return NOTIFY_DONE;
8958 }
8959 
8960 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
8961 				   unsigned long event, void *ptr)
8962 {
8963 	struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
8964 	struct net_device *dev = i6vi->i6vi_dev->dev;
8965 	struct mlxsw_sp *mlxsw_sp;
8966 	struct mlxsw_sp_rif *rif;
8967 	int err = 0;
8968 
8969 	mlxsw_sp = mlxsw_sp_lower_get(dev);
8970 	if (!mlxsw_sp)
8971 		return NOTIFY_DONE;
8972 
8973 	mutex_lock(&mlxsw_sp->router->lock);
8974 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8975 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
8976 		goto out;
8977 
8978 	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
8979 out:
8980 	mutex_unlock(&mlxsw_sp->router->lock);
8981 	return notifier_from_errno(err);
8982 }
8983 
8984 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8985 			     const char *mac, int mtu, u8 mac_profile)
8986 {
8987 	char ritr_pl[MLXSW_REG_RITR_LEN];
8988 	int err;
8989 
8990 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
8991 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
8992 	if (err)
8993 		return err;
8994 
8995 	mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
8996 	mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
8997 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile);
8998 	mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
8999 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9000 }
9001 
9002 static int
9003 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
9004 				  struct mlxsw_sp_rif *rif,
9005 				  struct netlink_ext_ack *extack)
9006 {
9007 	struct net_device *dev = rif->dev;
9008 	u8 old_mac_profile;
9009 	u16 fid_index;
9010 	int err;
9011 
9012 	fid_index = mlxsw_sp_fid_index(rif->fid);
9013 
9014 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
9015 	if (err)
9016 		return err;
9017 
9018 	old_mac_profile = rif->mac_profile_id;
9019 	err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr,
9020 					       extack);
9021 	if (err)
9022 		goto err_rif_mac_profile_replace;
9023 
9024 	err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
9025 				dev->mtu, rif->mac_profile_id);
9026 	if (err)
9027 		goto err_rif_edit;
9028 
9029 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
9030 	if (err)
9031 		goto err_rif_fdb_op;
9032 
9033 	if (rif->mtu != dev->mtu) {
9034 		struct mlxsw_sp_vr *vr;
9035 		int i;
9036 
9037 		/* The RIF is relevant only to its mr_table instance, as unlike
9038 		 * unicast routing, in multicast routing a RIF cannot be shared
9039 		 * between several multicast routing tables.
9040 		 */
9041 		vr = &mlxsw_sp->router->vrs[rif->vr_id];
9042 		for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
9043 			mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
9044 						   rif, dev->mtu);
9045 	}
9046 
9047 	ether_addr_copy(rif->addr, dev->dev_addr);
9048 	rif->mtu = dev->mtu;
9049 
9050 	netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
9051 
9052 	return 0;
9053 
9054 err_rif_fdb_op:
9055 	mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu,
9056 			  old_mac_profile);
9057 err_rif_edit:
9058 	mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack);
9059 err_rif_mac_profile_replace:
9060 	mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
9061 	return err;
9062 }
9063 
9064 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
9065 			    struct netdev_notifier_pre_changeaddr_info *info)
9066 {
9067 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9068 	struct mlxsw_sp_rif_mac_profile *profile;
9069 	struct netlink_ext_ack *extack;
9070 	u8 max_rif_mac_profiles;
9071 	u64 occ;
9072 
9073 	extack = netdev_notifier_info_to_extack(&info->info);
9074 
9075 	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr);
9076 	if (profile)
9077 		return 0;
9078 
9079 	max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
9080 	occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp);
9081 	if (occ < max_rif_mac_profiles)
9082 		return 0;
9083 
9084 	if (!mlxsw_sp_rif_mac_profile_is_shared(rif))
9085 		return 0;
9086 
9087 	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles");
9088 	return -ENOBUFS;
9089 }
9090 
9091 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event)
9092 {
9093 	switch (event) {
9094 	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9095 	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9096 	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9097 	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9098 		return true;
9099 	}
9100 
9101 	return false;
9102 }
9103 
9104 static int
9105 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif,
9106 					unsigned long event,
9107 					struct netdev_notifier_offload_xstats_info *info)
9108 {
9109 	switch (info->type) {
9110 	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9111 		break;
9112 	default:
9113 		return 0;
9114 	}
9115 
9116 	switch (event) {
9117 	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9118 		return mlxsw_sp_router_port_l3_stats_enable(rif);
9119 	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9120 		mlxsw_sp_router_port_l3_stats_disable(rif);
9121 		return 0;
9122 	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9123 		mlxsw_sp_router_port_l3_stats_report_used(rif, info);
9124 		return 0;
9125 	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9126 		return mlxsw_sp_router_port_l3_stats_report_delta(rif, info);
9127 	}
9128 
9129 	WARN_ON_ONCE(1);
9130 	return 0;
9131 }
9132 
9133 static int
9134 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp,
9135 				      struct net_device *dev,
9136 				      unsigned long event,
9137 				      struct netdev_notifier_offload_xstats_info *info)
9138 {
9139 	struct mlxsw_sp_rif *rif;
9140 
9141 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9142 	if (!rif)
9143 		return 0;
9144 
9145 	return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info);
9146 }
9147 
9148 static bool mlxsw_sp_is_router_event(unsigned long event)
9149 {
9150 	switch (event) {
9151 	case NETDEV_PRE_CHANGEADDR:
9152 	case NETDEV_CHANGEADDR:
9153 	case NETDEV_CHANGEMTU:
9154 		return true;
9155 	default:
9156 		return false;
9157 	}
9158 }
9159 
9160 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
9161 						unsigned long event, void *ptr)
9162 {
9163 	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
9164 	struct mlxsw_sp *mlxsw_sp;
9165 	struct mlxsw_sp_rif *rif;
9166 
9167 	mlxsw_sp = mlxsw_sp_lower_get(dev);
9168 	if (!mlxsw_sp)
9169 		return 0;
9170 
9171 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9172 	if (!rif)
9173 		return 0;
9174 
9175 	switch (event) {
9176 	case NETDEV_CHANGEMTU:
9177 	case NETDEV_CHANGEADDR:
9178 		return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack);
9179 	case NETDEV_PRE_CHANGEADDR:
9180 		return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
9181 	default:
9182 		WARN_ON_ONCE(1);
9183 		break;
9184 	}
9185 
9186 	return 0;
9187 }
9188 
9189 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
9190 				  struct net_device *l3_dev,
9191 				  struct netlink_ext_ack *extack)
9192 {
9193 	struct mlxsw_sp_rif *rif;
9194 
9195 	/* If netdev is already associated with a RIF, then we need to
9196 	 * destroy it and create a new one with the new virtual router ID.
9197 	 */
9198 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9199 	if (rif)
9200 		__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
9201 					  extack);
9202 
9203 	return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
9204 }
9205 
9206 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
9207 				    struct net_device *l3_dev)
9208 {
9209 	struct mlxsw_sp_rif *rif;
9210 
9211 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9212 	if (!rif)
9213 		return;
9214 	__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
9215 }
9216 
9217 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr)
9218 {
9219 	struct netdev_notifier_changeupper_info *info = ptr;
9220 
9221 	if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER)
9222 		return false;
9223 	return netif_is_l3_master(info->upper_dev);
9224 }
9225 
9226 static int
9227 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
9228 			     struct netdev_notifier_changeupper_info *info)
9229 {
9230 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
9231 	int err = 0;
9232 
9233 	/* We do not create a RIF for a macvlan, but only use it to
9234 	 * direct more MAC addresses to the router.
9235 	 */
9236 	if (!mlxsw_sp || netif_is_macvlan(l3_dev))
9237 		return 0;
9238 
9239 	switch (event) {
9240 	case NETDEV_PRECHANGEUPPER:
9241 		break;
9242 	case NETDEV_CHANGEUPPER:
9243 		if (info->linking) {
9244 			struct netlink_ext_ack *extack;
9245 
9246 			extack = netdev_notifier_info_to_extack(&info->info);
9247 			err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
9248 		} else {
9249 			mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
9250 		}
9251 		break;
9252 	}
9253 
9254 	return err;
9255 }
9256 
9257 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb,
9258 					   unsigned long event, void *ptr)
9259 {
9260 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
9261 	struct mlxsw_sp_router *router;
9262 	struct mlxsw_sp *mlxsw_sp;
9263 	int err = 0;
9264 
9265 	router = container_of(nb, struct mlxsw_sp_router, netdevice_nb);
9266 	mlxsw_sp = router->mlxsw_sp;
9267 
9268 	mutex_lock(&mlxsw_sp->router->lock);
9269 
9270 	if (mlxsw_sp_is_offload_xstats_event(event))
9271 		err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev,
9272 							    event, ptr);
9273 	else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev))
9274 		err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev,
9275 						       event, ptr);
9276 	else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev))
9277 		err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev,
9278 						       event, ptr);
9279 	else if (mlxsw_sp_is_router_event(event))
9280 		err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr);
9281 	else if (mlxsw_sp_is_vrf_event(event, ptr))
9282 		err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr);
9283 
9284 	mutex_unlock(&mlxsw_sp->router->lock);
9285 
9286 	return notifier_from_errno(err);
9287 }
9288 
9289 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
9290 					struct netdev_nested_priv *priv)
9291 {
9292 	struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
9293 
9294 	if (!netif_is_macvlan(dev))
9295 		return 0;
9296 
9297 	return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
9298 				   mlxsw_sp_fid_index(rif->fid), false);
9299 }
9300 
9301 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
9302 {
9303 	struct netdev_nested_priv priv = {
9304 		.data = (void *)rif,
9305 	};
9306 
9307 	if (!netif_is_macvlan_port(rif->dev))
9308 		return 0;
9309 
9310 	netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
9311 	return netdev_walk_all_upper_dev_rcu(rif->dev,
9312 					     __mlxsw_sp_rif_macvlan_flush, &priv);
9313 }
9314 
9315 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
9316 				       const struct mlxsw_sp_rif_params *params)
9317 {
9318 	struct mlxsw_sp_rif_subport *rif_subport;
9319 
9320 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
9321 	refcount_set(&rif_subport->ref_count, 1);
9322 	rif_subport->vid = params->vid;
9323 	rif_subport->lag = params->lag;
9324 	if (params->lag)
9325 		rif_subport->lag_id = params->lag_id;
9326 	else
9327 		rif_subport->system_port = params->system_port;
9328 }
9329 
9330 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
9331 {
9332 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9333 	struct mlxsw_sp_rif_subport *rif_subport;
9334 	char ritr_pl[MLXSW_REG_RITR_LEN];
9335 	u16 efid;
9336 
9337 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
9338 	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
9339 			    rif->rif_index, rif->vr_id, rif->dev->mtu);
9340 	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
9341 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
9342 	efid = mlxsw_sp_fid_index(rif->fid);
9343 	mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
9344 				  rif_subport->lag ? rif_subport->lag_id :
9345 						     rif_subport->system_port,
9346 				  efid, 0);
9347 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9348 }
9349 
9350 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif,
9351 					  struct netlink_ext_ack *extack)
9352 {
9353 	u8 mac_profile;
9354 	int err;
9355 
9356 	err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr,
9357 					   &mac_profile, extack);
9358 	if (err)
9359 		return err;
9360 	rif->mac_profile_id = mac_profile;
9361 
9362 	err = mlxsw_sp_rif_subport_op(rif, true);
9363 	if (err)
9364 		goto err_rif_subport_op;
9365 
9366 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9367 				  mlxsw_sp_fid_index(rif->fid), true);
9368 	if (err)
9369 		goto err_rif_fdb_op;
9370 
9371 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
9372 	if (err)
9373 		goto err_fid_rif_set;
9374 
9375 	return 0;
9376 
9377 err_fid_rif_set:
9378 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9379 			    mlxsw_sp_fid_index(rif->fid), false);
9380 err_rif_fdb_op:
9381 	mlxsw_sp_rif_subport_op(rif, false);
9382 err_rif_subport_op:
9383 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile);
9384 	return err;
9385 }
9386 
9387 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
9388 {
9389 	struct mlxsw_sp_fid *fid = rif->fid;
9390 
9391 	mlxsw_sp_fid_rif_unset(fid);
9392 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9393 			    mlxsw_sp_fid_index(fid), false);
9394 	mlxsw_sp_rif_macvlan_flush(rif);
9395 	mlxsw_sp_rif_subport_op(rif, false);
9396 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
9397 }
9398 
9399 static struct mlxsw_sp_fid *
9400 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
9401 			     struct netlink_ext_ack *extack)
9402 {
9403 	return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
9404 }
9405 
9406 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
9407 	.type			= MLXSW_SP_RIF_TYPE_SUBPORT,
9408 	.rif_size		= sizeof(struct mlxsw_sp_rif_subport),
9409 	.setup			= mlxsw_sp_rif_subport_setup,
9410 	.configure		= mlxsw_sp_rif_subport_configure,
9411 	.deconfigure		= mlxsw_sp_rif_subport_deconfigure,
9412 	.fid_get		= mlxsw_sp_rif_subport_fid_get,
9413 };
9414 
9415 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable)
9416 {
9417 	enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF;
9418 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9419 	char ritr_pl[MLXSW_REG_RITR_LEN];
9420 
9421 	mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
9422 			    rif->dev->mtu);
9423 	mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
9424 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
9425 	mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid);
9426 
9427 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9428 }
9429 
9430 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
9431 {
9432 	return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
9433 }
9434 
9435 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif,
9436 				      struct netlink_ext_ack *extack)
9437 {
9438 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9439 	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
9440 	u8 mac_profile;
9441 	int err;
9442 
9443 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
9444 					   &mac_profile, extack);
9445 	if (err)
9446 		return err;
9447 	rif->mac_profile_id = mac_profile;
9448 
9449 	err = mlxsw_sp_rif_fid_op(rif, fid_index, true);
9450 	if (err)
9451 		goto err_rif_fid_op;
9452 
9453 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9454 				     mlxsw_sp_router_port(mlxsw_sp), true);
9455 	if (err)
9456 		goto err_fid_mc_flood_set;
9457 
9458 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9459 				     mlxsw_sp_router_port(mlxsw_sp), true);
9460 	if (err)
9461 		goto err_fid_bc_flood_set;
9462 
9463 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9464 				  mlxsw_sp_fid_index(rif->fid), true);
9465 	if (err)
9466 		goto err_rif_fdb_op;
9467 
9468 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
9469 	if (err)
9470 		goto err_fid_rif_set;
9471 
9472 	return 0;
9473 
9474 err_fid_rif_set:
9475 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9476 			    mlxsw_sp_fid_index(rif->fid), false);
9477 err_rif_fdb_op:
9478 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9479 			       mlxsw_sp_router_port(mlxsw_sp), false);
9480 err_fid_bc_flood_set:
9481 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9482 			       mlxsw_sp_router_port(mlxsw_sp), false);
9483 err_fid_mc_flood_set:
9484 	mlxsw_sp_rif_fid_op(rif, fid_index, false);
9485 err_rif_fid_op:
9486 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
9487 	return err;
9488 }
9489 
9490 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
9491 {
9492 	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
9493 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9494 	struct mlxsw_sp_fid *fid = rif->fid;
9495 
9496 	mlxsw_sp_fid_rif_unset(fid);
9497 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9498 			    mlxsw_sp_fid_index(fid), false);
9499 	mlxsw_sp_rif_macvlan_flush(rif);
9500 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9501 			       mlxsw_sp_router_port(mlxsw_sp), false);
9502 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9503 			       mlxsw_sp_router_port(mlxsw_sp), false);
9504 	mlxsw_sp_rif_fid_op(rif, fid_index, false);
9505 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
9506 }
9507 
9508 static struct mlxsw_sp_fid *
9509 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
9510 			 struct netlink_ext_ack *extack)
9511 {
9512 	return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
9513 }
9514 
9515 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
9516 {
9517 	struct switchdev_notifier_fdb_info info = {};
9518 	struct net_device *dev;
9519 
9520 	dev = br_fdb_find_port(rif->dev, mac, 0);
9521 	if (!dev)
9522 		return;
9523 
9524 	info.addr = mac;
9525 	info.vid = 0;
9526 	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
9527 				 NULL);
9528 }
9529 
9530 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
9531 	.type			= MLXSW_SP_RIF_TYPE_FID,
9532 	.rif_size		= sizeof(struct mlxsw_sp_rif),
9533 	.configure		= mlxsw_sp_rif_fid_configure,
9534 	.deconfigure		= mlxsw_sp_rif_fid_deconfigure,
9535 	.fid_get		= mlxsw_sp_rif_fid_fid_get,
9536 	.fdb_del		= mlxsw_sp_rif_fid_fdb_del,
9537 };
9538 
9539 static struct mlxsw_sp_fid *
9540 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
9541 			  struct netlink_ext_ack *extack)
9542 {
9543 	struct net_device *br_dev;
9544 	u16 vid;
9545 	int err;
9546 
9547 	if (is_vlan_dev(rif->dev)) {
9548 		vid = vlan_dev_vlan_id(rif->dev);
9549 		br_dev = vlan_dev_real_dev(rif->dev);
9550 		if (WARN_ON(!netif_is_bridge_master(br_dev)))
9551 			return ERR_PTR(-EINVAL);
9552 	} else {
9553 		err = br_vlan_get_pvid(rif->dev, &vid);
9554 		if (err < 0 || !vid) {
9555 			NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
9556 			return ERR_PTR(-EINVAL);
9557 		}
9558 	}
9559 
9560 	return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
9561 }
9562 
9563 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
9564 {
9565 	struct switchdev_notifier_fdb_info info = {};
9566 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
9567 	struct net_device *br_dev;
9568 	struct net_device *dev;
9569 
9570 	br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
9571 	dev = br_fdb_find_port(br_dev, mac, vid);
9572 	if (!dev)
9573 		return;
9574 
9575 	info.addr = mac;
9576 	info.vid = vid;
9577 	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
9578 				 NULL);
9579 }
9580 
9581 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid,
9582 				bool enable)
9583 {
9584 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9585 	char ritr_pl[MLXSW_REG_RITR_LEN];
9586 
9587 	mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id,
9588 				    rif->dev->mtu, rif->dev->dev_addr,
9589 				    rif->mac_profile_id, vid, efid);
9590 
9591 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9592 }
9593 
9594 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid,
9595 				       struct netlink_ext_ack *extack)
9596 {
9597 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
9598 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9599 	u8 mac_profile;
9600 	int err;
9601 
9602 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
9603 					   &mac_profile, extack);
9604 	if (err)
9605 		return err;
9606 	rif->mac_profile_id = mac_profile;
9607 
9608 	err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true);
9609 	if (err)
9610 		goto err_rif_vlan_fid_op;
9611 
9612 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9613 				     mlxsw_sp_router_port(mlxsw_sp), true);
9614 	if (err)
9615 		goto err_fid_mc_flood_set;
9616 
9617 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9618 				     mlxsw_sp_router_port(mlxsw_sp), true);
9619 	if (err)
9620 		goto err_fid_bc_flood_set;
9621 
9622 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9623 				  mlxsw_sp_fid_index(rif->fid), true);
9624 	if (err)
9625 		goto err_rif_fdb_op;
9626 
9627 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
9628 	if (err)
9629 		goto err_fid_rif_set;
9630 
9631 	return 0;
9632 
9633 err_fid_rif_set:
9634 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9635 			    mlxsw_sp_fid_index(rif->fid), false);
9636 err_rif_fdb_op:
9637 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9638 			       mlxsw_sp_router_port(mlxsw_sp), false);
9639 err_fid_bc_flood_set:
9640 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9641 			       mlxsw_sp_router_port(mlxsw_sp), false);
9642 err_fid_mc_flood_set:
9643 	mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
9644 err_rif_vlan_fid_op:
9645 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
9646 	return err;
9647 }
9648 
9649 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
9650 {
9651 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
9652 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9653 
9654 	mlxsw_sp_fid_rif_unset(rif->fid);
9655 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
9656 			    mlxsw_sp_fid_index(rif->fid), false);
9657 	mlxsw_sp_rif_macvlan_flush(rif);
9658 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
9659 			       mlxsw_sp_router_port(mlxsw_sp), false);
9660 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
9661 			       mlxsw_sp_router_port(mlxsw_sp), false);
9662 	mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
9663 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
9664 }
9665 
9666 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif,
9667 					struct netlink_ext_ack *extack)
9668 {
9669 	return mlxsw_sp_rif_vlan_configure(rif, 0, extack);
9670 }
9671 
9672 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = {
9673 	.type			= MLXSW_SP_RIF_TYPE_VLAN,
9674 	.rif_size		= sizeof(struct mlxsw_sp_rif),
9675 	.configure		= mlxsw_sp1_rif_vlan_configure,
9676 	.deconfigure		= mlxsw_sp_rif_vlan_deconfigure,
9677 	.fid_get		= mlxsw_sp_rif_vlan_fid_get,
9678 	.fdb_del		= mlxsw_sp_rif_vlan_fdb_del,
9679 };
9680 
9681 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif,
9682 					struct netlink_ext_ack *extack)
9683 {
9684 	u16 efid = mlxsw_sp_fid_index(rif->fid);
9685 
9686 	return mlxsw_sp_rif_vlan_configure(rif, efid, extack);
9687 }
9688 
9689 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = {
9690 	.type			= MLXSW_SP_RIF_TYPE_VLAN,
9691 	.rif_size		= sizeof(struct mlxsw_sp_rif),
9692 	.configure		= mlxsw_sp2_rif_vlan_configure,
9693 	.deconfigure		= mlxsw_sp_rif_vlan_deconfigure,
9694 	.fid_get		= mlxsw_sp_rif_vlan_fid_get,
9695 	.fdb_del		= mlxsw_sp_rif_vlan_fdb_del,
9696 };
9697 
9698 static struct mlxsw_sp_rif_ipip_lb *
9699 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
9700 {
9701 	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
9702 }
9703 
9704 static void
9705 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
9706 			   const struct mlxsw_sp_rif_params *params)
9707 {
9708 	struct mlxsw_sp_rif_params_ipip_lb *params_lb;
9709 	struct mlxsw_sp_rif_ipip_lb *rif_lb;
9710 
9711 	params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
9712 				 common);
9713 	rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
9714 	rif_lb->lb_config = params_lb->lb_config;
9715 }
9716 
9717 static int
9718 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
9719 				struct netlink_ext_ack *extack)
9720 {
9721 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9722 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
9723 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9724 	struct mlxsw_sp_vr *ul_vr;
9725 	int err;
9726 
9727 	ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
9728 	if (IS_ERR(ul_vr))
9729 		return PTR_ERR(ul_vr);
9730 
9731 	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
9732 	if (err)
9733 		goto err_loopback_op;
9734 
9735 	lb_rif->ul_vr_id = ul_vr->id;
9736 	lb_rif->ul_rif_id = 0;
9737 	++ul_vr->rif_count;
9738 	return 0;
9739 
9740 err_loopback_op:
9741 	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
9742 	return err;
9743 }
9744 
9745 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
9746 {
9747 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9748 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9749 	struct mlxsw_sp_vr *ul_vr;
9750 
9751 	ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
9752 	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
9753 
9754 	--ul_vr->rif_count;
9755 	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
9756 }
9757 
9758 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
9759 	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
9760 	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
9761 	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
9762 	.configure		= mlxsw_sp1_rif_ipip_lb_configure,
9763 	.deconfigure		= mlxsw_sp1_rif_ipip_lb_deconfigure,
9764 };
9765 
9766 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
9767 	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
9768 	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp1_rif_vlan_ops,
9769 	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
9770 	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp1_rif_ipip_lb_ops,
9771 };
9772 
9773 static int
9774 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
9775 {
9776 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
9777 	char ritr_pl[MLXSW_REG_RITR_LEN];
9778 
9779 	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
9780 			    ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
9781 	mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
9782 					     MLXSW_REG_RITR_LOOPBACK_GENERIC);
9783 
9784 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9785 }
9786 
9787 static struct mlxsw_sp_rif *
9788 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
9789 		       struct netlink_ext_ack *extack)
9790 {
9791 	struct mlxsw_sp_rif *ul_rif;
9792 	u8 rif_entries = 1;
9793 	u16 rif_index;
9794 	int err;
9795 
9796 	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
9797 	if (err) {
9798 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
9799 		return ERR_PTR(err);
9800 	}
9801 
9802 	ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
9803 	if (!ul_rif) {
9804 		err = -ENOMEM;
9805 		goto err_rif_alloc;
9806 	}
9807 
9808 	mlxsw_sp->router->rifs[rif_index] = ul_rif;
9809 	ul_rif->mlxsw_sp = mlxsw_sp;
9810 	ul_rif->rif_entries = rif_entries;
9811 	err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
9812 	if (err)
9813 		goto ul_rif_op_err;
9814 
9815 	atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
9816 	return ul_rif;
9817 
9818 ul_rif_op_err:
9819 	mlxsw_sp->router->rifs[rif_index] = NULL;
9820 	kfree(ul_rif);
9821 err_rif_alloc:
9822 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
9823 	return ERR_PTR(err);
9824 }
9825 
9826 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
9827 {
9828 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
9829 	u8 rif_entries = ul_rif->rif_entries;
9830 	u16 rif_index = ul_rif->rif_index;
9831 
9832 	atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
9833 	mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
9834 	mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
9835 	kfree(ul_rif);
9836 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
9837 }
9838 
9839 static struct mlxsw_sp_rif *
9840 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
9841 		    struct netlink_ext_ack *extack)
9842 {
9843 	struct mlxsw_sp_vr *vr;
9844 	int err;
9845 
9846 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
9847 	if (IS_ERR(vr))
9848 		return ERR_CAST(vr);
9849 
9850 	if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
9851 		return vr->ul_rif;
9852 
9853 	vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
9854 	if (IS_ERR(vr->ul_rif)) {
9855 		err = PTR_ERR(vr->ul_rif);
9856 		goto err_ul_rif_create;
9857 	}
9858 
9859 	vr->rif_count++;
9860 	refcount_set(&vr->ul_rif_refcnt, 1);
9861 
9862 	return vr->ul_rif;
9863 
9864 err_ul_rif_create:
9865 	mlxsw_sp_vr_put(mlxsw_sp, vr);
9866 	return ERR_PTR(err);
9867 }
9868 
9869 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
9870 {
9871 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
9872 	struct mlxsw_sp_vr *vr;
9873 
9874 	vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
9875 
9876 	if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
9877 		return;
9878 
9879 	vr->rif_count--;
9880 	mlxsw_sp_ul_rif_destroy(ul_rif);
9881 	mlxsw_sp_vr_put(mlxsw_sp, vr);
9882 }
9883 
9884 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
9885 			       u16 *ul_rif_index)
9886 {
9887 	struct mlxsw_sp_rif *ul_rif;
9888 	int err = 0;
9889 
9890 	mutex_lock(&mlxsw_sp->router->lock);
9891 	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
9892 	if (IS_ERR(ul_rif)) {
9893 		err = PTR_ERR(ul_rif);
9894 		goto out;
9895 	}
9896 	*ul_rif_index = ul_rif->rif_index;
9897 out:
9898 	mutex_unlock(&mlxsw_sp->router->lock);
9899 	return err;
9900 }
9901 
9902 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
9903 {
9904 	struct mlxsw_sp_rif *ul_rif;
9905 
9906 	mutex_lock(&mlxsw_sp->router->lock);
9907 	ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
9908 	if (WARN_ON(!ul_rif))
9909 		goto out;
9910 
9911 	mlxsw_sp_ul_rif_put(ul_rif);
9912 out:
9913 	mutex_unlock(&mlxsw_sp->router->lock);
9914 }
9915 
9916 static int
9917 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
9918 				struct netlink_ext_ack *extack)
9919 {
9920 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9921 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
9922 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9923 	struct mlxsw_sp_rif *ul_rif;
9924 	int err;
9925 
9926 	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
9927 	if (IS_ERR(ul_rif))
9928 		return PTR_ERR(ul_rif);
9929 
9930 	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
9931 	if (err)
9932 		goto err_loopback_op;
9933 
9934 	lb_rif->ul_vr_id = 0;
9935 	lb_rif->ul_rif_id = ul_rif->rif_index;
9936 
9937 	return 0;
9938 
9939 err_loopback_op:
9940 	mlxsw_sp_ul_rif_put(ul_rif);
9941 	return err;
9942 }
9943 
9944 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
9945 {
9946 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
9947 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9948 	struct mlxsw_sp_rif *ul_rif;
9949 
9950 	ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
9951 	mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
9952 	mlxsw_sp_ul_rif_put(ul_rif);
9953 }
9954 
9955 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
9956 	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
9957 	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
9958 	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
9959 	.configure		= mlxsw_sp2_rif_ipip_lb_configure,
9960 	.deconfigure		= mlxsw_sp2_rif_ipip_lb_deconfigure,
9961 };
9962 
9963 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
9964 	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
9965 	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp2_rif_vlan_ops,
9966 	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
9967 	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp2_rif_ipip_lb_ops,
9968 };
9969 
9970 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp)
9971 {
9972 	struct gen_pool *rifs_table;
9973 	int err;
9974 
9975 	rifs_table = gen_pool_create(0, -1);
9976 	if (!rifs_table)
9977 		return -ENOMEM;
9978 
9979 	gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align,
9980 			  NULL);
9981 
9982 	err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET,
9983 			   MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1);
9984 	if (err)
9985 		goto err_gen_pool_add;
9986 
9987 	mlxsw_sp->router->rifs_table = rifs_table;
9988 
9989 	return 0;
9990 
9991 err_gen_pool_add:
9992 	gen_pool_destroy(rifs_table);
9993 	return err;
9994 }
9995 
9996 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp)
9997 {
9998 	gen_pool_destroy(mlxsw_sp->router->rifs_table);
9999 }
10000 
10001 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
10002 {
10003 	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10004 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10005 	struct mlxsw_core *core = mlxsw_sp->core;
10006 	int err;
10007 
10008 	if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES))
10009 		return -EIO;
10010 	mlxsw_sp->router->max_rif_mac_profile =
10011 		MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES);
10012 
10013 	mlxsw_sp->router->rifs = kcalloc(max_rifs,
10014 					 sizeof(struct mlxsw_sp_rif *),
10015 					 GFP_KERNEL);
10016 	if (!mlxsw_sp->router->rifs)
10017 		return -ENOMEM;
10018 
10019 	err = mlxsw_sp_rifs_table_init(mlxsw_sp);
10020 	if (err)
10021 		goto err_rifs_table_init;
10022 
10023 	idr_init(&mlxsw_sp->router->rif_mac_profiles_idr);
10024 	atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0);
10025 	atomic_set(&mlxsw_sp->router->rifs_count, 0);
10026 	devl_resource_occ_get_register(devlink,
10027 				       MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
10028 				       mlxsw_sp_rif_mac_profiles_occ_get,
10029 				       mlxsw_sp);
10030 	devl_resource_occ_get_register(devlink,
10031 				       MLXSW_SP_RESOURCE_RIFS,
10032 				       mlxsw_sp_rifs_occ_get,
10033 				       mlxsw_sp);
10034 
10035 	return 0;
10036 
10037 err_rifs_table_init:
10038 	kfree(mlxsw_sp->router->rifs);
10039 	return err;
10040 }
10041 
10042 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
10043 {
10044 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10045 	int i;
10046 
10047 	WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count));
10048 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
10049 		WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
10050 
10051 	devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS);
10052 	devl_resource_occ_get_unregister(devlink,
10053 					 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES);
10054 	WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr));
10055 	idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr);
10056 	mlxsw_sp_rifs_table_fini(mlxsw_sp);
10057 	kfree(mlxsw_sp->router->rifs);
10058 }
10059 
10060 static int
10061 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
10062 {
10063 	char tigcr_pl[MLXSW_REG_TIGCR_LEN];
10064 
10065 	mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
10066 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
10067 }
10068 
10069 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
10070 {
10071 	int err;
10072 
10073 	INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
10074 
10075 	err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
10076 	if (err)
10077 		return err;
10078 	err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
10079 	if (err)
10080 		return err;
10081 
10082 	return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
10083 }
10084 
10085 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp)
10086 {
10087 	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr;
10088 	return mlxsw_sp_ipips_init(mlxsw_sp);
10089 }
10090 
10091 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp)
10092 {
10093 	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr;
10094 	return mlxsw_sp_ipips_init(mlxsw_sp);
10095 }
10096 
10097 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
10098 {
10099 	WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
10100 }
10101 
10102 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
10103 {
10104 	struct mlxsw_sp_router *router;
10105 
10106 	/* Flush pending FIB notifications and then flush the device's
10107 	 * table before requesting another dump. The FIB notification
10108 	 * block is unregistered, so no need to take RTNL.
10109 	 */
10110 	mlxsw_core_flush_owq();
10111 	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
10112 	mlxsw_sp_router_fib_flush(router->mlxsw_sp);
10113 }
10114 
10115 #ifdef CONFIG_IP_ROUTE_MULTIPATH
10116 struct mlxsw_sp_mp_hash_config {
10117 	DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT);
10118 	DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT);
10119 	DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT);
10120 	DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT);
10121 	bool inc_parsing_depth;
10122 };
10123 
10124 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \
10125 	bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1)
10126 
10127 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \
10128 	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1)
10129 
10130 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \
10131 	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr)
10132 
10133 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config)
10134 {
10135 	unsigned long *inner_headers = config->inner_headers;
10136 	unsigned long *inner_fields = config->inner_fields;
10137 
10138 	/* IPv4 inner */
10139 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
10140 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
10141 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
10142 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
10143 	/* IPv6 inner */
10144 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
10145 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
10146 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
10147 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
10148 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
10149 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
10150 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
10151 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
10152 }
10153 
10154 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
10155 {
10156 	unsigned long *headers = config->headers;
10157 	unsigned long *fields = config->fields;
10158 
10159 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
10160 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
10161 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
10162 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
10163 }
10164 
10165 static void
10166 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config,
10167 			      u32 hash_fields)
10168 {
10169 	unsigned long *inner_headers = config->inner_headers;
10170 	unsigned long *inner_fields = config->inner_fields;
10171 
10172 	/* IPv4 Inner */
10173 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
10174 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
10175 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
10176 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
10177 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
10178 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
10179 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
10180 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL);
10181 	/* IPv6 inner */
10182 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
10183 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
10184 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) {
10185 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
10186 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
10187 	}
10188 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) {
10189 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
10190 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
10191 	}
10192 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
10193 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
10194 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
10195 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
10196 	/* L4 inner */
10197 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4);
10198 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6);
10199 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
10200 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT);
10201 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
10202 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT);
10203 }
10204 
10205 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp,
10206 				   struct mlxsw_sp_mp_hash_config *config)
10207 {
10208 	struct net *net = mlxsw_sp_net(mlxsw_sp);
10209 	unsigned long *headers = config->headers;
10210 	unsigned long *fields = config->fields;
10211 	u32 hash_fields;
10212 
10213 	switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) {
10214 	case 0:
10215 		mlxsw_sp_mp4_hash_outer_addr(config);
10216 		break;
10217 	case 1:
10218 		mlxsw_sp_mp4_hash_outer_addr(config);
10219 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
10220 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
10221 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
10222 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
10223 		break;
10224 	case 2:
10225 		/* Outer */
10226 		mlxsw_sp_mp4_hash_outer_addr(config);
10227 		/* Inner */
10228 		mlxsw_sp_mp_hash_inner_l3(config);
10229 		break;
10230 	case 3:
10231 		hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields);
10232 		/* Outer */
10233 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
10234 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
10235 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
10236 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
10237 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
10238 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
10239 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
10240 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
10241 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
10242 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
10243 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
10244 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
10245 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
10246 		/* Inner */
10247 		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
10248 		break;
10249 	}
10250 }
10251 
10252 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
10253 {
10254 	unsigned long *headers = config->headers;
10255 	unsigned long *fields = config->fields;
10256 
10257 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
10258 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
10259 	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
10260 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
10261 	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
10262 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
10263 }
10264 
10265 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp,
10266 				   struct mlxsw_sp_mp_hash_config *config)
10267 {
10268 	u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp));
10269 	unsigned long *headers = config->headers;
10270 	unsigned long *fields = config->fields;
10271 
10272 	switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) {
10273 	case 0:
10274 		mlxsw_sp_mp6_hash_outer_addr(config);
10275 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
10276 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
10277 		break;
10278 	case 1:
10279 		mlxsw_sp_mp6_hash_outer_addr(config);
10280 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
10281 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
10282 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
10283 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
10284 		break;
10285 	case 2:
10286 		/* Outer */
10287 		mlxsw_sp_mp6_hash_outer_addr(config);
10288 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
10289 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
10290 		/* Inner */
10291 		mlxsw_sp_mp_hash_inner_l3(config);
10292 		config->inc_parsing_depth = true;
10293 		break;
10294 	case 3:
10295 		/* Outer */
10296 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
10297 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
10298 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
10299 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) {
10300 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
10301 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
10302 		}
10303 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) {
10304 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
10305 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
10306 		}
10307 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
10308 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
10309 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
10310 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
10311 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
10312 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
10313 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
10314 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
10315 		/* Inner */
10316 		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
10317 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)
10318 			config->inc_parsing_depth = true;
10319 		break;
10320 	}
10321 }
10322 
10323 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp,
10324 						 bool old_inc_parsing_depth,
10325 						 bool new_inc_parsing_depth)
10326 {
10327 	int err;
10328 
10329 	if (!old_inc_parsing_depth && new_inc_parsing_depth) {
10330 		err = mlxsw_sp_parsing_depth_inc(mlxsw_sp);
10331 		if (err)
10332 			return err;
10333 		mlxsw_sp->router->inc_parsing_depth = true;
10334 	} else if (old_inc_parsing_depth && !new_inc_parsing_depth) {
10335 		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
10336 		mlxsw_sp->router->inc_parsing_depth = false;
10337 	}
10338 
10339 	return 0;
10340 }
10341 
10342 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
10343 {
10344 	bool old_inc_parsing_depth, new_inc_parsing_depth;
10345 	struct mlxsw_sp_mp_hash_config config = {};
10346 	char recr2_pl[MLXSW_REG_RECR2_LEN];
10347 	unsigned long bit;
10348 	u32 seed;
10349 	int err;
10350 
10351 	seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
10352 	mlxsw_reg_recr2_pack(recr2_pl, seed);
10353 	mlxsw_sp_mp4_hash_init(mlxsw_sp, &config);
10354 	mlxsw_sp_mp6_hash_init(mlxsw_sp, &config);
10355 
10356 	old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
10357 	new_inc_parsing_depth = config.inc_parsing_depth;
10358 	err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp,
10359 						    old_inc_parsing_depth,
10360 						    new_inc_parsing_depth);
10361 	if (err)
10362 		return err;
10363 
10364 	for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT)
10365 		mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1);
10366 	for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT)
10367 		mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1);
10368 	for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT)
10369 		mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1);
10370 	for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT)
10371 		mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1);
10372 
10373 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
10374 	if (err)
10375 		goto err_reg_write;
10376 
10377 	return 0;
10378 
10379 err_reg_write:
10380 	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth,
10381 					      old_inc_parsing_depth);
10382 	return err;
10383 }
10384 
10385 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
10386 {
10387 	bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
10388 
10389 	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth,
10390 					      false);
10391 }
10392 #else
10393 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
10394 {
10395 	return 0;
10396 }
10397 
10398 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
10399 {
10400 }
10401 #endif
10402 
10403 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
10404 {
10405 	char rdpm_pl[MLXSW_REG_RDPM_LEN];
10406 	unsigned int i;
10407 
10408 	MLXSW_REG_ZERO(rdpm, rdpm_pl);
10409 
10410 	/* HW is determining switch priority based on DSCP-bits, but the
10411 	 * kernel is still doing that based on the ToS. Since there's a
10412 	 * mismatch in bits we need to make sure to translate the right
10413 	 * value ToS would observe, skipping the 2 least-significant ECN bits.
10414 	 */
10415 	for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
10416 		mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
10417 
10418 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
10419 }
10420 
10421 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
10422 {
10423 	struct net *net = mlxsw_sp_net(mlxsw_sp);
10424 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
10425 	u64 max_rifs;
10426 	bool usp;
10427 
10428 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
10429 		return -EIO;
10430 	max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10431 	usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority);
10432 
10433 	mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
10434 	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
10435 	mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
10436 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
10437 }
10438 
10439 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
10440 {
10441 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
10442 
10443 	mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
10444 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
10445 }
10446 
10447 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp)
10448 {
10449 	u16 lb_rif_index;
10450 	int err;
10451 
10452 	/* Create a generic loopback RIF associated with the main table
10453 	 * (default VRF). Any table can be used, but the main table exists
10454 	 * anyway, so we do not waste resources.
10455 	 */
10456 	err = mlxsw_sp_router_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN,
10457 					 &lb_rif_index);
10458 	if (err)
10459 		return err;
10460 
10461 	mlxsw_sp->router->lb_rif_index = lb_rif_index;
10462 
10463 	return 0;
10464 }
10465 
10466 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
10467 {
10468 	mlxsw_sp_router_ul_rif_put(mlxsw_sp, mlxsw_sp->router->lb_rif_index);
10469 }
10470 
10471 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp)
10472 {
10473 	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges);
10474 
10475 	mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr;
10476 	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges;
10477 	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
10478 
10479 	return 0;
10480 }
10481 
10482 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = {
10483 	.init = mlxsw_sp1_router_init,
10484 	.ipips_init = mlxsw_sp1_ipips_init,
10485 };
10486 
10487 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp)
10488 {
10489 	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges);
10490 
10491 	mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr;
10492 	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges;
10493 	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
10494 
10495 	return 0;
10496 }
10497 
10498 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = {
10499 	.init = mlxsw_sp2_router_init,
10500 	.ipips_init = mlxsw_sp2_ipips_init,
10501 };
10502 
10503 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
10504 			 struct netlink_ext_ack *extack)
10505 {
10506 	struct mlxsw_sp_router *router;
10507 	int err;
10508 
10509 	router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
10510 	if (!router)
10511 		return -ENOMEM;
10512 	mutex_init(&router->lock);
10513 	mlxsw_sp->router = router;
10514 	router->mlxsw_sp = mlxsw_sp;
10515 
10516 	err = mlxsw_sp->router_ops->init(mlxsw_sp);
10517 	if (err)
10518 		goto err_router_ops_init;
10519 
10520 	INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list);
10521 	INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw,
10522 			  mlxsw_sp_nh_grp_activity_work);
10523 	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
10524 	err = __mlxsw_sp_router_init(mlxsw_sp);
10525 	if (err)
10526 		goto err_router_init;
10527 
10528 	err = mlxsw_sp_rifs_init(mlxsw_sp);
10529 	if (err)
10530 		goto err_rifs_init;
10531 
10532 	err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp);
10533 	if (err)
10534 		goto err_ipips_init;
10535 
10536 	err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
10537 			      &mlxsw_sp_nexthop_ht_params);
10538 	if (err)
10539 		goto err_nexthop_ht_init;
10540 
10541 	err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
10542 			      &mlxsw_sp_nexthop_group_ht_params);
10543 	if (err)
10544 		goto err_nexthop_group_ht_init;
10545 
10546 	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
10547 	err = mlxsw_sp_lpm_init(mlxsw_sp);
10548 	if (err)
10549 		goto err_lpm_init;
10550 
10551 	err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
10552 	if (err)
10553 		goto err_mr_init;
10554 
10555 	err = mlxsw_sp_vrs_init(mlxsw_sp);
10556 	if (err)
10557 		goto err_vrs_init;
10558 
10559 	err = mlxsw_sp_lb_rif_init(mlxsw_sp);
10560 	if (err)
10561 		goto err_lb_rif_init;
10562 
10563 	err = mlxsw_sp_neigh_init(mlxsw_sp);
10564 	if (err)
10565 		goto err_neigh_init;
10566 
10567 	err = mlxsw_sp_mp_hash_init(mlxsw_sp);
10568 	if (err)
10569 		goto err_mp_hash_init;
10570 
10571 	err = mlxsw_sp_dscp_init(mlxsw_sp);
10572 	if (err)
10573 		goto err_dscp_init;
10574 
10575 	router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
10576 	err = register_inetaddr_notifier(&router->inetaddr_nb);
10577 	if (err)
10578 		goto err_register_inetaddr_notifier;
10579 
10580 	router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
10581 	err = register_inet6addr_notifier(&router->inet6addr_nb);
10582 	if (err)
10583 		goto err_register_inet6addr_notifier;
10584 
10585 	mlxsw_sp->router->netevent_nb.notifier_call =
10586 		mlxsw_sp_router_netevent_event;
10587 	err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
10588 	if (err)
10589 		goto err_register_netevent_notifier;
10590 
10591 	mlxsw_sp->router->nexthop_nb.notifier_call =
10592 		mlxsw_sp_nexthop_obj_event;
10593 	err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
10594 					&mlxsw_sp->router->nexthop_nb,
10595 					extack);
10596 	if (err)
10597 		goto err_register_nexthop_notifier;
10598 
10599 	mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
10600 	err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
10601 				    &mlxsw_sp->router->fib_nb,
10602 				    mlxsw_sp_router_fib_dump_flush, extack);
10603 	if (err)
10604 		goto err_register_fib_notifier;
10605 
10606 	mlxsw_sp->router->netdevice_nb.notifier_call =
10607 		mlxsw_sp_router_netdevice_event;
10608 	err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
10609 					      &mlxsw_sp->router->netdevice_nb);
10610 	if (err)
10611 		goto err_register_netdev_notifier;
10612 
10613 	return 0;
10614 
10615 err_register_netdev_notifier:
10616 	unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
10617 				&mlxsw_sp->router->fib_nb);
10618 err_register_fib_notifier:
10619 	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
10620 				    &mlxsw_sp->router->nexthop_nb);
10621 err_register_nexthop_notifier:
10622 	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
10623 err_register_netevent_notifier:
10624 	unregister_inet6addr_notifier(&router->inet6addr_nb);
10625 err_register_inet6addr_notifier:
10626 	unregister_inetaddr_notifier(&router->inetaddr_nb);
10627 err_register_inetaddr_notifier:
10628 	mlxsw_core_flush_owq();
10629 err_dscp_init:
10630 	mlxsw_sp_mp_hash_fini(mlxsw_sp);
10631 err_mp_hash_init:
10632 	mlxsw_sp_neigh_fini(mlxsw_sp);
10633 err_neigh_init:
10634 	mlxsw_sp_lb_rif_fini(mlxsw_sp);
10635 err_lb_rif_init:
10636 	mlxsw_sp_vrs_fini(mlxsw_sp);
10637 err_vrs_init:
10638 	mlxsw_sp_mr_fini(mlxsw_sp);
10639 err_mr_init:
10640 	mlxsw_sp_lpm_fini(mlxsw_sp);
10641 err_lpm_init:
10642 	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
10643 err_nexthop_group_ht_init:
10644 	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
10645 err_nexthop_ht_init:
10646 	mlxsw_sp_ipips_fini(mlxsw_sp);
10647 err_ipips_init:
10648 	mlxsw_sp_rifs_fini(mlxsw_sp);
10649 err_rifs_init:
10650 	__mlxsw_sp_router_fini(mlxsw_sp);
10651 err_router_init:
10652 	cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
10653 err_router_ops_init:
10654 	mutex_destroy(&mlxsw_sp->router->lock);
10655 	kfree(mlxsw_sp->router);
10656 	return err;
10657 }
10658 
10659 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
10660 {
10661 	unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
10662 					  &mlxsw_sp->router->netdevice_nb);
10663 	unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp),
10664 				&mlxsw_sp->router->fib_nb);
10665 	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
10666 				    &mlxsw_sp->router->nexthop_nb);
10667 	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
10668 	unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
10669 	unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
10670 	mlxsw_core_flush_owq();
10671 	mlxsw_sp_mp_hash_fini(mlxsw_sp);
10672 	mlxsw_sp_neigh_fini(mlxsw_sp);
10673 	mlxsw_sp_lb_rif_fini(mlxsw_sp);
10674 	mlxsw_sp_vrs_fini(mlxsw_sp);
10675 	mlxsw_sp_mr_fini(mlxsw_sp);
10676 	mlxsw_sp_lpm_fini(mlxsw_sp);
10677 	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
10678 	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
10679 	mlxsw_sp_ipips_fini(mlxsw_sp);
10680 	mlxsw_sp_rifs_fini(mlxsw_sp);
10681 	__mlxsw_sp_router_fini(mlxsw_sp);
10682 	cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
10683 	mutex_destroy(&mlxsw_sp->router->lock);
10684 	kfree(mlxsw_sp->router);
10685 }
10686