1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/rhashtable.h>
7 #include <linux/bitops.h>
8 #include <linux/in6.h>
9 #include <linux/notifier.h>
10 #include <linux/inetdevice.h>
11 #include <linux/netdevice.h>
12 #include <linux/if_bridge.h>
13 #include <linux/socket.h>
14 #include <linux/route.h>
15 #include <linux/gcd.h>
16 #include <linux/if_macvlan.h>
17 #include <linux/refcount.h>
18 #include <linux/jhash.h>
19 #include <linux/net_namespace.h>
20 #include <linux/mutex.h>
21 #include <linux/genalloc.h>
22 #include <net/netevent.h>
23 #include <net/neighbour.h>
24 #include <net/arp.h>
25 #include <net/inet_dscp.h>
26 #include <net/ip_fib.h>
27 #include <net/ip6_fib.h>
28 #include <net/nexthop.h>
29 #include <net/fib_rules.h>
30 #include <net/ip_tunnels.h>
31 #include <net/l3mdev.h>
32 #include <net/addrconf.h>
33 #include <net/ndisc.h>
34 #include <net/ipv6.h>
35 #include <net/fib_notifier.h>
36 #include <net/switchdev.h>
37 
38 #include "spectrum.h"
39 #include "core.h"
40 #include "reg.h"
41 #include "spectrum_cnt.h"
42 #include "spectrum_dpipe.h"
43 #include "spectrum_ipip.h"
44 #include "spectrum_mr.h"
45 #include "spectrum_mr_tcam.h"
46 #include "spectrum_router.h"
47 #include "spectrum_span.h"
48 
49 struct mlxsw_sp_fib;
50 struct mlxsw_sp_vr;
51 struct mlxsw_sp_lpm_tree;
52 struct mlxsw_sp_rif_ops;
53 
54 struct mlxsw_sp_crif_key {
55 	struct net_device *dev;
56 };
57 
58 struct mlxsw_sp_crif {
59 	struct mlxsw_sp_crif_key key;
60 	struct rhash_head ht_node;
61 	bool can_destroy;
62 	struct list_head nexthop_list;
63 	struct mlxsw_sp_rif *rif;
64 };
65 
66 static const struct rhashtable_params mlxsw_sp_crif_ht_params = {
67 	.key_offset = offsetof(struct mlxsw_sp_crif, key),
68 	.key_len = sizeof_field(struct mlxsw_sp_crif, key),
69 	.head_offset = offsetof(struct mlxsw_sp_crif, ht_node),
70 };
71 
72 struct mlxsw_sp_rif {
73 	struct mlxsw_sp_crif *crif; /* NULL for underlay RIF */
74 	netdevice_tracker dev_tracker;
75 	struct list_head neigh_list;
76 	struct mlxsw_sp_fid *fid;
77 	unsigned char addr[ETH_ALEN];
78 	int mtu;
79 	u16 rif_index;
80 	u8 mac_profile_id;
81 	u8 rif_entries;
82 	u16 vr_id;
83 	const struct mlxsw_sp_rif_ops *ops;
84 	struct mlxsw_sp *mlxsw_sp;
85 
86 	unsigned int counter_ingress;
87 	bool counter_ingress_valid;
88 	unsigned int counter_egress;
89 	bool counter_egress_valid;
90 };
91 
92 static struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
93 {
94 	if (!rif->crif)
95 		return NULL;
96 	return rif->crif->key.dev;
97 }
98 
99 struct mlxsw_sp_rif_params {
100 	struct net_device *dev;
101 	union {
102 		u16 system_port;
103 		u16 lag_id;
104 	};
105 	u16 vid;
106 	bool lag;
107 	bool double_entry;
108 };
109 
110 struct mlxsw_sp_rif_subport {
111 	struct mlxsw_sp_rif common;
112 	refcount_t ref_count;
113 	union {
114 		u16 system_port;
115 		u16 lag_id;
116 	};
117 	u16 vid;
118 	bool lag;
119 };
120 
121 struct mlxsw_sp_rif_ipip_lb {
122 	struct mlxsw_sp_rif common;
123 	struct mlxsw_sp_rif_ipip_lb_config lb_config;
124 	u16 ul_vr_id;	/* Spectrum-1. */
125 	u16 ul_rif_id;	/* Spectrum-2+. */
126 };
127 
128 struct mlxsw_sp_rif_params_ipip_lb {
129 	struct mlxsw_sp_rif_params common;
130 	struct mlxsw_sp_rif_ipip_lb_config lb_config;
131 };
132 
133 struct mlxsw_sp_rif_ops {
134 	enum mlxsw_sp_rif_type type;
135 	size_t rif_size;
136 
137 	void (*setup)(struct mlxsw_sp_rif *rif,
138 		      const struct mlxsw_sp_rif_params *params);
139 	int (*configure)(struct mlxsw_sp_rif *rif,
140 			 struct netlink_ext_ack *extack);
141 	void (*deconfigure)(struct mlxsw_sp_rif *rif);
142 	struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
143 					 const struct mlxsw_sp_rif_params *params,
144 					 struct netlink_ext_ack *extack);
145 	void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
146 };
147 
148 struct mlxsw_sp_rif_mac_profile {
149 	unsigned char mac_prefix[ETH_ALEN];
150 	refcount_t ref_count;
151 	u8 id;
152 };
153 
154 struct mlxsw_sp_router_ops {
155 	int (*init)(struct mlxsw_sp *mlxsw_sp);
156 	int (*ipips_init)(struct mlxsw_sp *mlxsw_sp);
157 };
158 
159 static struct mlxsw_sp_rif *
160 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
161 			 const struct net_device *dev);
162 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
163 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
164 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
165 				  struct mlxsw_sp_lpm_tree *lpm_tree);
166 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
167 				     const struct mlxsw_sp_fib *fib,
168 				     u8 tree_id);
169 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
170 				       const struct mlxsw_sp_fib *fib);
171 
172 static unsigned int *
173 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
174 			   enum mlxsw_sp_rif_counter_dir dir)
175 {
176 	switch (dir) {
177 	case MLXSW_SP_RIF_COUNTER_EGRESS:
178 		return &rif->counter_egress;
179 	case MLXSW_SP_RIF_COUNTER_INGRESS:
180 		return &rif->counter_ingress;
181 	}
182 	return NULL;
183 }
184 
185 static bool
186 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
187 			       enum mlxsw_sp_rif_counter_dir dir)
188 {
189 	switch (dir) {
190 	case MLXSW_SP_RIF_COUNTER_EGRESS:
191 		return rif->counter_egress_valid;
192 	case MLXSW_SP_RIF_COUNTER_INGRESS:
193 		return rif->counter_ingress_valid;
194 	}
195 	return false;
196 }
197 
198 static void
199 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
200 			       enum mlxsw_sp_rif_counter_dir dir,
201 			       bool valid)
202 {
203 	switch (dir) {
204 	case MLXSW_SP_RIF_COUNTER_EGRESS:
205 		rif->counter_egress_valid = valid;
206 		break;
207 	case MLXSW_SP_RIF_COUNTER_INGRESS:
208 		rif->counter_ingress_valid = valid;
209 		break;
210 	}
211 }
212 
213 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
214 				     unsigned int counter_index, bool enable,
215 				     enum mlxsw_sp_rif_counter_dir dir)
216 {
217 	char ritr_pl[MLXSW_REG_RITR_LEN];
218 	bool is_egress = false;
219 	int err;
220 
221 	if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
222 		is_egress = true;
223 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
224 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
225 	if (err)
226 		return err;
227 
228 	mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
229 				    is_egress);
230 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
231 }
232 
233 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
234 				   struct mlxsw_sp_rif *rif,
235 				   enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
236 {
237 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
238 	unsigned int *p_counter_index;
239 	bool valid;
240 	int err;
241 
242 	valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
243 	if (!valid)
244 		return -EINVAL;
245 
246 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
247 	if (!p_counter_index)
248 		return -EINVAL;
249 	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
250 			     MLXSW_REG_RICNT_OPCODE_NOP);
251 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
252 	if (err)
253 		return err;
254 	*cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
255 	return 0;
256 }
257 
258 struct mlxsw_sp_rif_counter_set_basic {
259 	u64 good_unicast_packets;
260 	u64 good_multicast_packets;
261 	u64 good_broadcast_packets;
262 	u64 good_unicast_bytes;
263 	u64 good_multicast_bytes;
264 	u64 good_broadcast_bytes;
265 	u64 error_packets;
266 	u64 discard_packets;
267 	u64 error_bytes;
268 	u64 discard_bytes;
269 };
270 
271 static int
272 mlxsw_sp_rif_counter_fetch_clear(struct mlxsw_sp_rif *rif,
273 				 enum mlxsw_sp_rif_counter_dir dir,
274 				 struct mlxsw_sp_rif_counter_set_basic *set)
275 {
276 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
277 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
278 	unsigned int *p_counter_index;
279 	int err;
280 
281 	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
282 		return -EINVAL;
283 
284 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
285 	if (!p_counter_index)
286 		return -EINVAL;
287 
288 	mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
289 			     MLXSW_REG_RICNT_OPCODE_CLEAR);
290 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
291 	if (err)
292 		return err;
293 
294 	if (!set)
295 		return 0;
296 
297 #define MLXSW_SP_RIF_COUNTER_EXTRACT(NAME)				\
298 		(set->NAME = mlxsw_reg_ricnt_ ## NAME ## _get(ricnt_pl))
299 
300 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_packets);
301 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_packets);
302 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_packets);
303 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_unicast_bytes);
304 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_multicast_bytes);
305 	MLXSW_SP_RIF_COUNTER_EXTRACT(good_broadcast_bytes);
306 	MLXSW_SP_RIF_COUNTER_EXTRACT(error_packets);
307 	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_packets);
308 	MLXSW_SP_RIF_COUNTER_EXTRACT(error_bytes);
309 	MLXSW_SP_RIF_COUNTER_EXTRACT(discard_bytes);
310 
311 #undef MLXSW_SP_RIF_COUNTER_EXTRACT
312 
313 	return 0;
314 }
315 
316 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
317 				      unsigned int counter_index)
318 {
319 	char ricnt_pl[MLXSW_REG_RICNT_LEN];
320 
321 	mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
322 			     MLXSW_REG_RICNT_OPCODE_CLEAR);
323 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
324 }
325 
326 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp_rif *rif,
327 			       enum mlxsw_sp_rif_counter_dir dir)
328 {
329 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
330 	unsigned int *p_counter_index;
331 	int err;
332 
333 	if (mlxsw_sp_rif_counter_valid_get(rif, dir))
334 		return 0;
335 
336 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
337 	if (!p_counter_index)
338 		return -EINVAL;
339 
340 	err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
341 				     p_counter_index);
342 	if (err)
343 		return err;
344 
345 	err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
346 	if (err)
347 		goto err_counter_clear;
348 
349 	err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
350 					*p_counter_index, true, dir);
351 	if (err)
352 		goto err_counter_edit;
353 	mlxsw_sp_rif_counter_valid_set(rif, dir, true);
354 	return 0;
355 
356 err_counter_edit:
357 err_counter_clear:
358 	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
359 			      *p_counter_index);
360 	return err;
361 }
362 
363 void mlxsw_sp_rif_counter_free(struct mlxsw_sp_rif *rif,
364 			       enum mlxsw_sp_rif_counter_dir dir)
365 {
366 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
367 	unsigned int *p_counter_index;
368 
369 	if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
370 		return;
371 
372 	p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
373 	if (WARN_ON(!p_counter_index))
374 		return;
375 	mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
376 				  *p_counter_index, false, dir);
377 	mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
378 			      *p_counter_index);
379 	mlxsw_sp_rif_counter_valid_set(rif, dir, false);
380 }
381 
382 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
383 {
384 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
385 	struct devlink *devlink;
386 
387 	devlink = priv_to_devlink(mlxsw_sp->core);
388 	if (!devlink_dpipe_table_counter_enabled(devlink,
389 						 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
390 		return;
391 	mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
392 }
393 
394 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
395 {
396 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
397 }
398 
399 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
400 
401 struct mlxsw_sp_prefix_usage {
402 	DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
403 };
404 
405 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
406 	for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
407 
408 static bool
409 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
410 			 struct mlxsw_sp_prefix_usage *prefix_usage2)
411 {
412 	return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
413 }
414 
415 static void
416 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
417 			  struct mlxsw_sp_prefix_usage *prefix_usage2)
418 {
419 	memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
420 }
421 
422 static void
423 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
424 			  unsigned char prefix_len)
425 {
426 	set_bit(prefix_len, prefix_usage->b);
427 }
428 
429 static void
430 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
431 			    unsigned char prefix_len)
432 {
433 	clear_bit(prefix_len, prefix_usage->b);
434 }
435 
436 struct mlxsw_sp_fib_key {
437 	unsigned char addr[sizeof(struct in6_addr)];
438 	unsigned char prefix_len;
439 };
440 
441 enum mlxsw_sp_fib_entry_type {
442 	MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
443 	MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
444 	MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
445 	MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
446 	MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE,
447 
448 	/* This is a special case of local delivery, where a packet should be
449 	 * decapsulated on reception. Note that there is no corresponding ENCAP,
450 	 * because that's a type of next hop, not of FIB entry. (There can be
451 	 * several next hops in a REMOTE entry, and some of them may be
452 	 * encapsulating entries.)
453 	 */
454 	MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
455 	MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
456 };
457 
458 struct mlxsw_sp_nexthop_group_info;
459 struct mlxsw_sp_nexthop_group;
460 struct mlxsw_sp_fib_entry;
461 
462 struct mlxsw_sp_fib_node {
463 	struct mlxsw_sp_fib_entry *fib_entry;
464 	struct list_head list;
465 	struct rhash_head ht_node;
466 	struct mlxsw_sp_fib *fib;
467 	struct mlxsw_sp_fib_key key;
468 };
469 
470 struct mlxsw_sp_fib_entry_decap {
471 	struct mlxsw_sp_ipip_entry *ipip_entry;
472 	u32 tunnel_index;
473 };
474 
475 struct mlxsw_sp_fib_entry {
476 	struct mlxsw_sp_fib_node *fib_node;
477 	enum mlxsw_sp_fib_entry_type type;
478 	struct list_head nexthop_group_node;
479 	struct mlxsw_sp_nexthop_group *nh_group;
480 	struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
481 };
482 
483 struct mlxsw_sp_fib4_entry {
484 	struct mlxsw_sp_fib_entry common;
485 	struct fib_info *fi;
486 	u32 tb_id;
487 	dscp_t dscp;
488 	u8 type;
489 };
490 
491 struct mlxsw_sp_fib6_entry {
492 	struct mlxsw_sp_fib_entry common;
493 	struct list_head rt6_list;
494 	unsigned int nrt6;
495 };
496 
497 struct mlxsw_sp_rt6 {
498 	struct list_head list;
499 	struct fib6_info *rt;
500 };
501 
502 struct mlxsw_sp_lpm_tree {
503 	u8 id; /* tree ID */
504 	refcount_t ref_count;
505 	enum mlxsw_sp_l3proto proto;
506 	unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
507 	struct mlxsw_sp_prefix_usage prefix_usage;
508 };
509 
510 struct mlxsw_sp_fib {
511 	struct rhashtable ht;
512 	struct list_head node_list;
513 	struct mlxsw_sp_vr *vr;
514 	struct mlxsw_sp_lpm_tree *lpm_tree;
515 	enum mlxsw_sp_l3proto proto;
516 };
517 
518 struct mlxsw_sp_vr {
519 	u16 id; /* virtual router ID */
520 	u32 tb_id; /* kernel fib table id */
521 	unsigned int rif_count;
522 	struct mlxsw_sp_fib *fib4;
523 	struct mlxsw_sp_fib *fib6;
524 	struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
525 	struct mlxsw_sp_rif *ul_rif;
526 	refcount_t ul_rif_refcnt;
527 };
528 
529 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
530 
531 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
532 						struct mlxsw_sp_vr *vr,
533 						enum mlxsw_sp_l3proto proto)
534 {
535 	struct mlxsw_sp_lpm_tree *lpm_tree;
536 	struct mlxsw_sp_fib *fib;
537 	int err;
538 
539 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
540 	fib = kzalloc(sizeof(*fib), GFP_KERNEL);
541 	if (!fib)
542 		return ERR_PTR(-ENOMEM);
543 	err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
544 	if (err)
545 		goto err_rhashtable_init;
546 	INIT_LIST_HEAD(&fib->node_list);
547 	fib->proto = proto;
548 	fib->vr = vr;
549 	fib->lpm_tree = lpm_tree;
550 	mlxsw_sp_lpm_tree_hold(lpm_tree);
551 	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
552 	if (err)
553 		goto err_lpm_tree_bind;
554 	return fib;
555 
556 err_lpm_tree_bind:
557 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
558 err_rhashtable_init:
559 	kfree(fib);
560 	return ERR_PTR(err);
561 }
562 
563 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
564 				 struct mlxsw_sp_fib *fib)
565 {
566 	mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
567 	mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
568 	WARN_ON(!list_empty(&fib->node_list));
569 	rhashtable_destroy(&fib->ht);
570 	kfree(fib);
571 }
572 
573 static struct mlxsw_sp_lpm_tree *
574 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
575 {
576 	static struct mlxsw_sp_lpm_tree *lpm_tree;
577 	int i;
578 
579 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
580 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
581 		if (refcount_read(&lpm_tree->ref_count) == 0)
582 			return lpm_tree;
583 	}
584 	return NULL;
585 }
586 
587 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
588 				   struct mlxsw_sp_lpm_tree *lpm_tree)
589 {
590 	char ralta_pl[MLXSW_REG_RALTA_LEN];
591 
592 	mlxsw_reg_ralta_pack(ralta_pl, true,
593 			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
594 			     lpm_tree->id);
595 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
596 }
597 
598 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
599 				   struct mlxsw_sp_lpm_tree *lpm_tree)
600 {
601 	char ralta_pl[MLXSW_REG_RALTA_LEN];
602 
603 	mlxsw_reg_ralta_pack(ralta_pl, false,
604 			     (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
605 			     lpm_tree->id);
606 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
607 }
608 
609 static int
610 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
611 				  struct mlxsw_sp_prefix_usage *prefix_usage,
612 				  struct mlxsw_sp_lpm_tree *lpm_tree)
613 {
614 	char ralst_pl[MLXSW_REG_RALST_LEN];
615 	u8 root_bin = 0;
616 	u8 prefix;
617 	u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
618 
619 	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
620 		root_bin = prefix;
621 
622 	mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
623 	mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
624 		if (prefix == 0)
625 			continue;
626 		mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
627 					 MLXSW_REG_RALST_BIN_NO_CHILD);
628 		last_prefix = prefix;
629 	}
630 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
631 }
632 
633 static struct mlxsw_sp_lpm_tree *
634 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
635 			 struct mlxsw_sp_prefix_usage *prefix_usage,
636 			 enum mlxsw_sp_l3proto proto)
637 {
638 	struct mlxsw_sp_lpm_tree *lpm_tree;
639 	int err;
640 
641 	lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
642 	if (!lpm_tree)
643 		return ERR_PTR(-EBUSY);
644 	lpm_tree->proto = proto;
645 	err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
646 	if (err)
647 		return ERR_PTR(err);
648 
649 	err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
650 						lpm_tree);
651 	if (err)
652 		goto err_left_struct_set;
653 	memcpy(&lpm_tree->prefix_usage, prefix_usage,
654 	       sizeof(lpm_tree->prefix_usage));
655 	memset(&lpm_tree->prefix_ref_count, 0,
656 	       sizeof(lpm_tree->prefix_ref_count));
657 	refcount_set(&lpm_tree->ref_count, 1);
658 	return lpm_tree;
659 
660 err_left_struct_set:
661 	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
662 	return ERR_PTR(err);
663 }
664 
665 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
666 				      struct mlxsw_sp_lpm_tree *lpm_tree)
667 {
668 	mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
669 }
670 
671 static struct mlxsw_sp_lpm_tree *
672 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
673 		      struct mlxsw_sp_prefix_usage *prefix_usage,
674 		      enum mlxsw_sp_l3proto proto)
675 {
676 	struct mlxsw_sp_lpm_tree *lpm_tree;
677 	int i;
678 
679 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
680 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
681 		if (refcount_read(&lpm_tree->ref_count) &&
682 		    lpm_tree->proto == proto &&
683 		    mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
684 					     prefix_usage)) {
685 			mlxsw_sp_lpm_tree_hold(lpm_tree);
686 			return lpm_tree;
687 		}
688 	}
689 	return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
690 }
691 
692 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
693 {
694 	refcount_inc(&lpm_tree->ref_count);
695 }
696 
697 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
698 				  struct mlxsw_sp_lpm_tree *lpm_tree)
699 {
700 	if (!refcount_dec_and_test(&lpm_tree->ref_count))
701 		return;
702 	mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
703 }
704 
705 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
706 
707 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
708 {
709 	struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
710 	struct mlxsw_sp_lpm_tree *lpm_tree;
711 	u64 max_trees;
712 	int err, i;
713 
714 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
715 		return -EIO;
716 
717 	max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
718 	mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
719 	mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
720 					     sizeof(struct mlxsw_sp_lpm_tree),
721 					     GFP_KERNEL);
722 	if (!mlxsw_sp->router->lpm.trees)
723 		return -ENOMEM;
724 
725 	for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
726 		lpm_tree = &mlxsw_sp->router->lpm.trees[i];
727 		lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
728 	}
729 
730 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
731 					 MLXSW_SP_L3_PROTO_IPV4);
732 	if (IS_ERR(lpm_tree)) {
733 		err = PTR_ERR(lpm_tree);
734 		goto err_ipv4_tree_get;
735 	}
736 	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
737 
738 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
739 					 MLXSW_SP_L3_PROTO_IPV6);
740 	if (IS_ERR(lpm_tree)) {
741 		err = PTR_ERR(lpm_tree);
742 		goto err_ipv6_tree_get;
743 	}
744 	mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
745 
746 	return 0;
747 
748 err_ipv6_tree_get:
749 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
750 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
751 err_ipv4_tree_get:
752 	kfree(mlxsw_sp->router->lpm.trees);
753 	return err;
754 }
755 
756 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
757 {
758 	struct mlxsw_sp_lpm_tree *lpm_tree;
759 
760 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
761 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
762 
763 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
764 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
765 
766 	kfree(mlxsw_sp->router->lpm.trees);
767 }
768 
769 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
770 {
771 	return !!vr->fib4 || !!vr->fib6 ||
772 	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
773 	       !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
774 }
775 
776 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
777 {
778 	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
779 	struct mlxsw_sp_vr *vr;
780 	int i;
781 
782 	for (i = 0; i < max_vrs; i++) {
783 		vr = &mlxsw_sp->router->vrs[i];
784 		if (!mlxsw_sp_vr_is_used(vr))
785 			return vr;
786 	}
787 	return NULL;
788 }
789 
790 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
791 				     const struct mlxsw_sp_fib *fib, u8 tree_id)
792 {
793 	char raltb_pl[MLXSW_REG_RALTB_LEN];
794 
795 	mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
796 			     (enum mlxsw_reg_ralxx_protocol) fib->proto,
797 			     tree_id);
798 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
799 }
800 
801 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
802 				       const struct mlxsw_sp_fib *fib)
803 {
804 	char raltb_pl[MLXSW_REG_RALTB_LEN];
805 
806 	/* Bind to tree 0 which is default */
807 	mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
808 			     (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
809 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
810 }
811 
812 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
813 {
814 	/* For our purpose, squash main, default and local tables into one */
815 	if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
816 		tb_id = RT_TABLE_MAIN;
817 	return tb_id;
818 }
819 
820 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
821 					    u32 tb_id)
822 {
823 	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
824 	struct mlxsw_sp_vr *vr;
825 	int i;
826 
827 	tb_id = mlxsw_sp_fix_tb_id(tb_id);
828 
829 	for (i = 0; i < max_vrs; i++) {
830 		vr = &mlxsw_sp->router->vrs[i];
831 		if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
832 			return vr;
833 	}
834 	return NULL;
835 }
836 
837 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
838 				u16 *vr_id)
839 {
840 	struct mlxsw_sp_vr *vr;
841 	int err = 0;
842 
843 	mutex_lock(&mlxsw_sp->router->lock);
844 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
845 	if (!vr) {
846 		err = -ESRCH;
847 		goto out;
848 	}
849 	*vr_id = vr->id;
850 out:
851 	mutex_unlock(&mlxsw_sp->router->lock);
852 	return err;
853 }
854 
855 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
856 					    enum mlxsw_sp_l3proto proto)
857 {
858 	switch (proto) {
859 	case MLXSW_SP_L3_PROTO_IPV4:
860 		return vr->fib4;
861 	case MLXSW_SP_L3_PROTO_IPV6:
862 		return vr->fib6;
863 	}
864 	return NULL;
865 }
866 
867 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
868 					      u32 tb_id,
869 					      struct netlink_ext_ack *extack)
870 {
871 	struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
872 	struct mlxsw_sp_fib *fib4;
873 	struct mlxsw_sp_fib *fib6;
874 	struct mlxsw_sp_vr *vr;
875 	int err;
876 
877 	vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
878 	if (!vr) {
879 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
880 		return ERR_PTR(-EBUSY);
881 	}
882 	fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
883 	if (IS_ERR(fib4))
884 		return ERR_CAST(fib4);
885 	fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
886 	if (IS_ERR(fib6)) {
887 		err = PTR_ERR(fib6);
888 		goto err_fib6_create;
889 	}
890 	mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
891 					     MLXSW_SP_L3_PROTO_IPV4);
892 	if (IS_ERR(mr4_table)) {
893 		err = PTR_ERR(mr4_table);
894 		goto err_mr4_table_create;
895 	}
896 	mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
897 					     MLXSW_SP_L3_PROTO_IPV6);
898 	if (IS_ERR(mr6_table)) {
899 		err = PTR_ERR(mr6_table);
900 		goto err_mr6_table_create;
901 	}
902 
903 	vr->fib4 = fib4;
904 	vr->fib6 = fib6;
905 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
906 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
907 	vr->tb_id = tb_id;
908 	return vr;
909 
910 err_mr6_table_create:
911 	mlxsw_sp_mr_table_destroy(mr4_table);
912 err_mr4_table_create:
913 	mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
914 err_fib6_create:
915 	mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
916 	return ERR_PTR(err);
917 }
918 
919 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
920 				struct mlxsw_sp_vr *vr)
921 {
922 	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
923 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
924 	mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
925 	vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
926 	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
927 	vr->fib6 = NULL;
928 	mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
929 	vr->fib4 = NULL;
930 }
931 
932 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
933 					   struct netlink_ext_ack *extack)
934 {
935 	struct mlxsw_sp_vr *vr;
936 
937 	tb_id = mlxsw_sp_fix_tb_id(tb_id);
938 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
939 	if (!vr)
940 		vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
941 	return vr;
942 }
943 
944 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
945 {
946 	if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
947 	    list_empty(&vr->fib6->node_list) &&
948 	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
949 	    mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
950 		mlxsw_sp_vr_destroy(mlxsw_sp, vr);
951 }
952 
953 static bool
954 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
955 				    enum mlxsw_sp_l3proto proto, u8 tree_id)
956 {
957 	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
958 
959 	if (!mlxsw_sp_vr_is_used(vr))
960 		return false;
961 	if (fib->lpm_tree->id == tree_id)
962 		return true;
963 	return false;
964 }
965 
966 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
967 					struct mlxsw_sp_fib *fib,
968 					struct mlxsw_sp_lpm_tree *new_tree)
969 {
970 	struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
971 	int err;
972 
973 	fib->lpm_tree = new_tree;
974 	mlxsw_sp_lpm_tree_hold(new_tree);
975 	err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
976 	if (err)
977 		goto err_tree_bind;
978 	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
979 	return 0;
980 
981 err_tree_bind:
982 	mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
983 	fib->lpm_tree = old_tree;
984 	return err;
985 }
986 
987 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
988 					 struct mlxsw_sp_fib *fib,
989 					 struct mlxsw_sp_lpm_tree *new_tree)
990 {
991 	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
992 	enum mlxsw_sp_l3proto proto = fib->proto;
993 	struct mlxsw_sp_lpm_tree *old_tree;
994 	u8 old_id, new_id = new_tree->id;
995 	struct mlxsw_sp_vr *vr;
996 	int i, err;
997 
998 	old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
999 	old_id = old_tree->id;
1000 
1001 	for (i = 0; i < max_vrs; i++) {
1002 		vr = &mlxsw_sp->router->vrs[i];
1003 		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
1004 			continue;
1005 		err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1006 						   mlxsw_sp_vr_fib(vr, proto),
1007 						   new_tree);
1008 		if (err)
1009 			goto err_tree_replace;
1010 	}
1011 
1012 	memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
1013 	       sizeof(new_tree->prefix_ref_count));
1014 	mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
1015 	mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
1016 
1017 	return 0;
1018 
1019 err_tree_replace:
1020 	for (i--; i >= 0; i--) {
1021 		if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
1022 			continue;
1023 		mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
1024 					     mlxsw_sp_vr_fib(vr, proto),
1025 					     old_tree);
1026 	}
1027 	return err;
1028 }
1029 
1030 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
1031 {
1032 	struct mlxsw_sp_vr *vr;
1033 	u64 max_vrs;
1034 	int i;
1035 
1036 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
1037 		return -EIO;
1038 
1039 	max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
1040 	mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
1041 					GFP_KERNEL);
1042 	if (!mlxsw_sp->router->vrs)
1043 		return -ENOMEM;
1044 
1045 	for (i = 0; i < max_vrs; i++) {
1046 		vr = &mlxsw_sp->router->vrs[i];
1047 		vr->id = i;
1048 	}
1049 
1050 	return 0;
1051 }
1052 
1053 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
1054 
1055 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
1056 {
1057 	/* At this stage we're guaranteed not to have new incoming
1058 	 * FIB notifications and the work queue is free from FIBs
1059 	 * sitting on top of mlxsw netdevs. However, we can still
1060 	 * have other FIBs queued. Flush the queue before flushing
1061 	 * the device's tables. No need for locks, as we're the only
1062 	 * writer.
1063 	 */
1064 	mlxsw_core_flush_owq();
1065 	mlxsw_sp_router_fib_flush(mlxsw_sp);
1066 	kfree(mlxsw_sp->router->vrs);
1067 }
1068 
1069 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1070 {
1071 	struct net_device *d;
1072 	u32 tb_id;
1073 
1074 	rcu_read_lock();
1075 	d = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1076 	if (d)
1077 		tb_id = l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1078 	else
1079 		tb_id = RT_TABLE_MAIN;
1080 	rcu_read_unlock();
1081 
1082 	return tb_id;
1083 }
1084 
1085 static void
1086 mlxsw_sp_crif_init(struct mlxsw_sp_crif *crif, struct net_device *dev)
1087 {
1088 	crif->key.dev = dev;
1089 	INIT_LIST_HEAD(&crif->nexthop_list);
1090 }
1091 
1092 static struct mlxsw_sp_crif *
1093 mlxsw_sp_crif_alloc(struct net_device *dev)
1094 {
1095 	struct mlxsw_sp_crif *crif;
1096 
1097 	crif = kzalloc(sizeof(*crif), GFP_KERNEL);
1098 	if (!crif)
1099 		return NULL;
1100 
1101 	mlxsw_sp_crif_init(crif, dev);
1102 	return crif;
1103 }
1104 
1105 static void mlxsw_sp_crif_free(struct mlxsw_sp_crif *crif)
1106 {
1107 	if (WARN_ON(crif->rif))
1108 		return;
1109 
1110 	WARN_ON(!list_empty(&crif->nexthop_list));
1111 	kfree(crif);
1112 }
1113 
1114 static int mlxsw_sp_crif_insert(struct mlxsw_sp_router *router,
1115 				struct mlxsw_sp_crif *crif)
1116 {
1117 	return rhashtable_insert_fast(&router->crif_ht, &crif->ht_node,
1118 				      mlxsw_sp_crif_ht_params);
1119 }
1120 
1121 static void mlxsw_sp_crif_remove(struct mlxsw_sp_router *router,
1122 				 struct mlxsw_sp_crif *crif)
1123 {
1124 	rhashtable_remove_fast(&router->crif_ht, &crif->ht_node,
1125 			       mlxsw_sp_crif_ht_params);
1126 }
1127 
1128 static struct mlxsw_sp_crif *
1129 mlxsw_sp_crif_lookup(struct mlxsw_sp_router *router,
1130 		     const struct net_device *dev)
1131 {
1132 	struct mlxsw_sp_crif_key key = {
1133 		.dev = (struct net_device *)dev,
1134 	};
1135 
1136 	return rhashtable_lookup_fast(&router->crif_ht, &key,
1137 				      mlxsw_sp_crif_ht_params);
1138 }
1139 
1140 static struct mlxsw_sp_rif *
1141 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1142 		    const struct mlxsw_sp_rif_params *params,
1143 		    struct netlink_ext_ack *extack);
1144 
1145 static struct mlxsw_sp_rif_ipip_lb *
1146 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1147 				enum mlxsw_sp_ipip_type ipipt,
1148 				struct net_device *ol_dev,
1149 				struct netlink_ext_ack *extack)
1150 {
1151 	struct mlxsw_sp_rif_params_ipip_lb lb_params;
1152 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1153 	struct mlxsw_sp_rif *rif;
1154 
1155 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1156 	lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1157 		.common.dev = ol_dev,
1158 		.common.lag = false,
1159 		.common.double_entry = ipip_ops->double_rif_entry,
1160 		.lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1161 	};
1162 
1163 	rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1164 	if (IS_ERR(rif))
1165 		return ERR_CAST(rif);
1166 	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1167 }
1168 
1169 static struct mlxsw_sp_ipip_entry *
1170 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1171 			  enum mlxsw_sp_ipip_type ipipt,
1172 			  struct net_device *ol_dev)
1173 {
1174 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1175 	struct mlxsw_sp_ipip_entry *ipip_entry;
1176 	struct mlxsw_sp_ipip_entry *ret = NULL;
1177 	int err;
1178 
1179 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1180 	ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1181 	if (!ipip_entry)
1182 		return ERR_PTR(-ENOMEM);
1183 
1184 	ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1185 							    ol_dev, NULL);
1186 	if (IS_ERR(ipip_entry->ol_lb)) {
1187 		ret = ERR_CAST(ipip_entry->ol_lb);
1188 		goto err_ol_ipip_lb_create;
1189 	}
1190 
1191 	ipip_entry->ipipt = ipipt;
1192 	ipip_entry->ol_dev = ol_dev;
1193 	ipip_entry->parms = ipip_ops->parms_init(ol_dev);
1194 
1195 	err = ipip_ops->rem_ip_addr_set(mlxsw_sp, ipip_entry);
1196 	if (err) {
1197 		ret = ERR_PTR(err);
1198 		goto err_rem_ip_addr_set;
1199 	}
1200 
1201 	return ipip_entry;
1202 
1203 err_rem_ip_addr_set:
1204 	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1205 err_ol_ipip_lb_create:
1206 	kfree(ipip_entry);
1207 	return ret;
1208 }
1209 
1210 static void mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp *mlxsw_sp,
1211 					struct mlxsw_sp_ipip_entry *ipip_entry)
1212 {
1213 	const struct mlxsw_sp_ipip_ops *ipip_ops =
1214 		mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1215 
1216 	ipip_ops->rem_ip_addr_unset(mlxsw_sp, ipip_entry);
1217 	mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1218 	kfree(ipip_entry);
1219 }
1220 
1221 static bool
1222 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1223 				  const enum mlxsw_sp_l3proto ul_proto,
1224 				  union mlxsw_sp_l3addr saddr,
1225 				  u32 ul_tb_id,
1226 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1227 {
1228 	u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1229 	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1230 	union mlxsw_sp_l3addr tun_saddr;
1231 
1232 	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1233 		return false;
1234 
1235 	tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1236 	return tun_ul_tb_id == ul_tb_id &&
1237 	       mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1238 }
1239 
1240 static int mlxsw_sp_ipip_decap_parsing_depth_inc(struct mlxsw_sp *mlxsw_sp,
1241 						 enum mlxsw_sp_ipip_type ipipt)
1242 {
1243 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1244 
1245 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1246 
1247 	/* Not all tunnels require to increase the default pasing depth
1248 	 * (96 bytes).
1249 	 */
1250 	if (ipip_ops->inc_parsing_depth)
1251 		return mlxsw_sp_parsing_depth_inc(mlxsw_sp);
1252 
1253 	return 0;
1254 }
1255 
1256 static void mlxsw_sp_ipip_decap_parsing_depth_dec(struct mlxsw_sp *mlxsw_sp,
1257 						  enum mlxsw_sp_ipip_type ipipt)
1258 {
1259 	const struct mlxsw_sp_ipip_ops *ipip_ops =
1260 		mlxsw_sp->router->ipip_ops_arr[ipipt];
1261 
1262 	if (ipip_ops->inc_parsing_depth)
1263 		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
1264 }
1265 
1266 static int
1267 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1268 			      struct mlxsw_sp_fib_entry *fib_entry,
1269 			      struct mlxsw_sp_ipip_entry *ipip_entry)
1270 {
1271 	u32 tunnel_index;
1272 	int err;
1273 
1274 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1275 				  1, &tunnel_index);
1276 	if (err)
1277 		return err;
1278 
1279 	err = mlxsw_sp_ipip_decap_parsing_depth_inc(mlxsw_sp,
1280 						    ipip_entry->ipipt);
1281 	if (err)
1282 		goto err_parsing_depth_inc;
1283 
1284 	ipip_entry->decap_fib_entry = fib_entry;
1285 	fib_entry->decap.ipip_entry = ipip_entry;
1286 	fib_entry->decap.tunnel_index = tunnel_index;
1287 
1288 	return 0;
1289 
1290 err_parsing_depth_inc:
1291 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
1292 			   fib_entry->decap.tunnel_index);
1293 	return err;
1294 }
1295 
1296 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1297 					  struct mlxsw_sp_fib_entry *fib_entry)
1298 {
1299 	enum mlxsw_sp_ipip_type ipipt = fib_entry->decap.ipip_entry->ipipt;
1300 
1301 	/* Unlink this node from the IPIP entry that it's the decap entry of. */
1302 	fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1303 	fib_entry->decap.ipip_entry = NULL;
1304 	mlxsw_sp_ipip_decap_parsing_depth_dec(mlxsw_sp, ipipt);
1305 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1306 			   1, fib_entry->decap.tunnel_index);
1307 }
1308 
1309 static struct mlxsw_sp_fib_node *
1310 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1311 			 size_t addr_len, unsigned char prefix_len);
1312 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1313 				     struct mlxsw_sp_fib_entry *fib_entry);
1314 
1315 static void
1316 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1317 				 struct mlxsw_sp_ipip_entry *ipip_entry)
1318 {
1319 	struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1320 
1321 	mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1322 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1323 
1324 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1325 }
1326 
1327 static void
1328 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1329 				  struct mlxsw_sp_ipip_entry *ipip_entry,
1330 				  struct mlxsw_sp_fib_entry *decap_fib_entry)
1331 {
1332 	if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1333 					  ipip_entry))
1334 		return;
1335 	decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1336 
1337 	if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1338 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1339 }
1340 
1341 static struct mlxsw_sp_fib_entry *
1342 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1343 				     enum mlxsw_sp_l3proto proto,
1344 				     const union mlxsw_sp_l3addr *addr,
1345 				     enum mlxsw_sp_fib_entry_type type)
1346 {
1347 	struct mlxsw_sp_fib_node *fib_node;
1348 	unsigned char addr_prefix_len;
1349 	struct mlxsw_sp_fib *fib;
1350 	struct mlxsw_sp_vr *vr;
1351 	const void *addrp;
1352 	size_t addr_len;
1353 	u32 addr4;
1354 
1355 	vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1356 	if (!vr)
1357 		return NULL;
1358 	fib = mlxsw_sp_vr_fib(vr, proto);
1359 
1360 	switch (proto) {
1361 	case MLXSW_SP_L3_PROTO_IPV4:
1362 		addr4 = be32_to_cpu(addr->addr4);
1363 		addrp = &addr4;
1364 		addr_len = 4;
1365 		addr_prefix_len = 32;
1366 		break;
1367 	case MLXSW_SP_L3_PROTO_IPV6:
1368 		addrp = &addr->addr6;
1369 		addr_len = 16;
1370 		addr_prefix_len = 128;
1371 		break;
1372 	default:
1373 		WARN_ON(1);
1374 		return NULL;
1375 	}
1376 
1377 	fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1378 					    addr_prefix_len);
1379 	if (!fib_node || fib_node->fib_entry->type != type)
1380 		return NULL;
1381 
1382 	return fib_node->fib_entry;
1383 }
1384 
1385 /* Given an IPIP entry, find the corresponding decap route. */
1386 static struct mlxsw_sp_fib_entry *
1387 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1388 			       struct mlxsw_sp_ipip_entry *ipip_entry)
1389 {
1390 	static struct mlxsw_sp_fib_node *fib_node;
1391 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1392 	unsigned char saddr_prefix_len;
1393 	union mlxsw_sp_l3addr saddr;
1394 	struct mlxsw_sp_fib *ul_fib;
1395 	struct mlxsw_sp_vr *ul_vr;
1396 	const void *saddrp;
1397 	size_t saddr_len;
1398 	u32 ul_tb_id;
1399 	u32 saddr4;
1400 
1401 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1402 
1403 	ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1404 	ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1405 	if (!ul_vr)
1406 		return NULL;
1407 
1408 	ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1409 	saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1410 					   ipip_entry->ol_dev);
1411 
1412 	switch (ipip_ops->ul_proto) {
1413 	case MLXSW_SP_L3_PROTO_IPV4:
1414 		saddr4 = be32_to_cpu(saddr.addr4);
1415 		saddrp = &saddr4;
1416 		saddr_len = 4;
1417 		saddr_prefix_len = 32;
1418 		break;
1419 	case MLXSW_SP_L3_PROTO_IPV6:
1420 		saddrp = &saddr.addr6;
1421 		saddr_len = 16;
1422 		saddr_prefix_len = 128;
1423 		break;
1424 	default:
1425 		WARN_ON(1);
1426 		return NULL;
1427 	}
1428 
1429 	fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1430 					    saddr_prefix_len);
1431 	if (!fib_node ||
1432 	    fib_node->fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1433 		return NULL;
1434 
1435 	return fib_node->fib_entry;
1436 }
1437 
1438 static struct mlxsw_sp_ipip_entry *
1439 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1440 			   enum mlxsw_sp_ipip_type ipipt,
1441 			   struct net_device *ol_dev)
1442 {
1443 	struct mlxsw_sp_ipip_entry *ipip_entry;
1444 
1445 	ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1446 	if (IS_ERR(ipip_entry))
1447 		return ipip_entry;
1448 
1449 	list_add_tail(&ipip_entry->ipip_list_node,
1450 		      &mlxsw_sp->router->ipip_list);
1451 
1452 	return ipip_entry;
1453 }
1454 
1455 static void
1456 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1457 			    struct mlxsw_sp_ipip_entry *ipip_entry)
1458 {
1459 	list_del(&ipip_entry->ipip_list_node);
1460 	mlxsw_sp_ipip_entry_dealloc(mlxsw_sp, ipip_entry);
1461 }
1462 
1463 static bool
1464 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1465 				  const struct net_device *ul_dev,
1466 				  enum mlxsw_sp_l3proto ul_proto,
1467 				  union mlxsw_sp_l3addr ul_dip,
1468 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1469 {
1470 	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1471 	enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1472 
1473 	if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1474 		return false;
1475 
1476 	return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1477 						 ul_tb_id, ipip_entry);
1478 }
1479 
1480 /* Given decap parameters, find the corresponding IPIP entry. */
1481 static struct mlxsw_sp_ipip_entry *
1482 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp, int ul_dev_ifindex,
1483 				  enum mlxsw_sp_l3proto ul_proto,
1484 				  union mlxsw_sp_l3addr ul_dip)
1485 {
1486 	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1487 	struct net_device *ul_dev;
1488 
1489 	rcu_read_lock();
1490 
1491 	ul_dev = dev_get_by_index_rcu(mlxsw_sp_net(mlxsw_sp), ul_dev_ifindex);
1492 	if (!ul_dev)
1493 		goto out_unlock;
1494 
1495 	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1496 			    ipip_list_node)
1497 		if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1498 						      ul_proto, ul_dip,
1499 						      ipip_entry))
1500 			goto out_unlock;
1501 
1502 	rcu_read_unlock();
1503 
1504 	return NULL;
1505 
1506 out_unlock:
1507 	rcu_read_unlock();
1508 	return ipip_entry;
1509 }
1510 
1511 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1512 				      const struct net_device *dev,
1513 				      enum mlxsw_sp_ipip_type *p_type)
1514 {
1515 	struct mlxsw_sp_router *router = mlxsw_sp->router;
1516 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1517 	enum mlxsw_sp_ipip_type ipipt;
1518 
1519 	for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1520 		ipip_ops = router->ipip_ops_arr[ipipt];
1521 		if (dev->type == ipip_ops->dev_type) {
1522 			if (p_type)
1523 				*p_type = ipipt;
1524 			return true;
1525 		}
1526 	}
1527 	return false;
1528 }
1529 
1530 static bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1531 				       const struct net_device *dev)
1532 {
1533 	return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1534 }
1535 
1536 static struct mlxsw_sp_ipip_entry *
1537 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1538 				   const struct net_device *ol_dev)
1539 {
1540 	struct mlxsw_sp_ipip_entry *ipip_entry;
1541 
1542 	list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1543 			    ipip_list_node)
1544 		if (ipip_entry->ol_dev == ol_dev)
1545 			return ipip_entry;
1546 
1547 	return NULL;
1548 }
1549 
1550 static struct mlxsw_sp_ipip_entry *
1551 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1552 				   const struct net_device *ul_dev,
1553 				   struct mlxsw_sp_ipip_entry *start)
1554 {
1555 	struct mlxsw_sp_ipip_entry *ipip_entry;
1556 
1557 	ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1558 					ipip_list_node);
1559 	list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1560 				     ipip_list_node) {
1561 		struct net_device *ol_dev = ipip_entry->ol_dev;
1562 		struct net_device *ipip_ul_dev;
1563 
1564 		rcu_read_lock();
1565 		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1566 		rcu_read_unlock();
1567 
1568 		if (ipip_ul_dev == ul_dev)
1569 			return ipip_entry;
1570 	}
1571 
1572 	return NULL;
1573 }
1574 
1575 static bool mlxsw_sp_netdev_is_ipip_ul(struct mlxsw_sp *mlxsw_sp,
1576 				       const struct net_device *dev)
1577 {
1578 	return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1579 }
1580 
1581 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1582 						const struct net_device *ol_dev,
1583 						enum mlxsw_sp_ipip_type ipipt)
1584 {
1585 	const struct mlxsw_sp_ipip_ops *ops
1586 		= mlxsw_sp->router->ipip_ops_arr[ipipt];
1587 
1588 	return ops->can_offload(mlxsw_sp, ol_dev);
1589 }
1590 
1591 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1592 						struct net_device *ol_dev)
1593 {
1594 	enum mlxsw_sp_ipip_type ipipt = MLXSW_SP_IPIP_TYPE_MAX;
1595 	struct mlxsw_sp_ipip_entry *ipip_entry;
1596 	enum mlxsw_sp_l3proto ul_proto;
1597 	union mlxsw_sp_l3addr saddr;
1598 	u32 ul_tb_id;
1599 
1600 	mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1601 	if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1602 		ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1603 		ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1604 		saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1605 		if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1606 							  saddr, ul_tb_id,
1607 							  NULL)) {
1608 			ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1609 								ol_dev);
1610 			if (IS_ERR(ipip_entry))
1611 				return PTR_ERR(ipip_entry);
1612 		}
1613 	}
1614 
1615 	return 0;
1616 }
1617 
1618 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1619 						   struct net_device *ol_dev)
1620 {
1621 	struct mlxsw_sp_ipip_entry *ipip_entry;
1622 
1623 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1624 	if (ipip_entry)
1625 		mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1626 }
1627 
1628 static void
1629 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1630 				struct mlxsw_sp_ipip_entry *ipip_entry)
1631 {
1632 	struct mlxsw_sp_fib_entry *decap_fib_entry;
1633 
1634 	decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1635 	if (decap_fib_entry)
1636 		mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1637 						  decap_fib_entry);
1638 }
1639 
1640 static int
1641 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1642 			u16 ul_rif_id, bool enable)
1643 {
1644 	struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1645 	struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common);
1646 	enum mlxsw_reg_ritr_loopback_ipip_options ipip_options;
1647 	struct mlxsw_sp_rif *rif = &lb_rif->common;
1648 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1649 	char ritr_pl[MLXSW_REG_RITR_LEN];
1650 	struct in6_addr *saddr6;
1651 	u32 saddr4;
1652 
1653 	ipip_options = MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET;
1654 	switch (lb_cf.ul_protocol) {
1655 	case MLXSW_SP_L3_PROTO_IPV4:
1656 		saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1657 		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1658 				    rif->rif_index, rif->vr_id, dev->mtu);
1659 		mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1660 						   ipip_options, ul_vr_id,
1661 						   ul_rif_id, saddr4,
1662 						   lb_cf.okey);
1663 		break;
1664 
1665 	case MLXSW_SP_L3_PROTO_IPV6:
1666 		saddr6 = &lb_cf.saddr.addr6;
1667 		mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1668 				    rif->rif_index, rif->vr_id, dev->mtu);
1669 		mlxsw_reg_ritr_loopback_ipip6_pack(ritr_pl, lb_cf.lb_ipipt,
1670 						   ipip_options, ul_vr_id,
1671 						   ul_rif_id, saddr6,
1672 						   lb_cf.okey);
1673 		break;
1674 	}
1675 
1676 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1677 }
1678 
1679 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1680 						 struct net_device *ol_dev)
1681 {
1682 	struct mlxsw_sp_ipip_entry *ipip_entry;
1683 	struct mlxsw_sp_rif_ipip_lb *lb_rif;
1684 	int err = 0;
1685 
1686 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1687 	if (ipip_entry) {
1688 		lb_rif = ipip_entry->ol_lb;
1689 		err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1690 					      lb_rif->ul_rif_id, true);
1691 		if (err)
1692 			goto out;
1693 		lb_rif->common.mtu = ol_dev->mtu;
1694 	}
1695 
1696 out:
1697 	return err;
1698 }
1699 
1700 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1701 						struct net_device *ol_dev)
1702 {
1703 	struct mlxsw_sp_ipip_entry *ipip_entry;
1704 
1705 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1706 	if (ipip_entry)
1707 		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1708 }
1709 
1710 static void
1711 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1712 				  struct mlxsw_sp_ipip_entry *ipip_entry)
1713 {
1714 	if (ipip_entry->decap_fib_entry)
1715 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1716 }
1717 
1718 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1719 						  struct net_device *ol_dev)
1720 {
1721 	struct mlxsw_sp_ipip_entry *ipip_entry;
1722 
1723 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1724 	if (ipip_entry)
1725 		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1726 }
1727 
1728 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1729 					struct mlxsw_sp_rif *rif);
1730 
1731 static void mlxsw_sp_rif_migrate_destroy(struct mlxsw_sp *mlxsw_sp,
1732 					 struct mlxsw_sp_rif *old_rif,
1733 					 struct mlxsw_sp_rif *new_rif,
1734 					 bool migrate_nhs)
1735 {
1736 	struct mlxsw_sp_crif *crif = old_rif->crif;
1737 	struct mlxsw_sp_crif mock_crif = {};
1738 
1739 	if (migrate_nhs)
1740 		mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
1741 
1742 	/* Plant a mock CRIF so that destroying the old RIF doesn't unoffload
1743 	 * our nexthops and IPIP tunnels, and doesn't sever the crif->rif link.
1744 	 */
1745 	mlxsw_sp_crif_init(&mock_crif, crif->key.dev);
1746 	old_rif->crif = &mock_crif;
1747 	mock_crif.rif = old_rif;
1748 	mlxsw_sp_rif_destroy(old_rif);
1749 }
1750 
1751 static int
1752 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1753 				 struct mlxsw_sp_ipip_entry *ipip_entry,
1754 				 bool keep_encap,
1755 				 struct netlink_ext_ack *extack)
1756 {
1757 	struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1758 	struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1759 
1760 	new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1761 						     ipip_entry->ipipt,
1762 						     ipip_entry->ol_dev,
1763 						     extack);
1764 	if (IS_ERR(new_lb_rif))
1765 		return PTR_ERR(new_lb_rif);
1766 	ipip_entry->ol_lb = new_lb_rif;
1767 
1768 	mlxsw_sp_rif_migrate_destroy(mlxsw_sp, &old_lb_rif->common,
1769 				     &new_lb_rif->common, keep_encap);
1770 	return 0;
1771 }
1772 
1773 /**
1774  * __mlxsw_sp_ipip_entry_update_tunnel - Update offload related to IPIP entry.
1775  * @mlxsw_sp: mlxsw_sp.
1776  * @ipip_entry: IPIP entry.
1777  * @recreate_loopback: Recreates the associated loopback RIF.
1778  * @keep_encap: Updates next hops that use the tunnel netdevice. This is only
1779  *              relevant when recreate_loopback is true.
1780  * @update_nexthops: Updates next hops, keeping the current loopback RIF. This
1781  *                   is only relevant when recreate_loopback is false.
1782  * @extack: extack.
1783  *
1784  * Return: Non-zero value on failure.
1785  */
1786 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1787 					struct mlxsw_sp_ipip_entry *ipip_entry,
1788 					bool recreate_loopback,
1789 					bool keep_encap,
1790 					bool update_nexthops,
1791 					struct netlink_ext_ack *extack)
1792 {
1793 	int err;
1794 
1795 	/* RIFs can't be edited, so to update loopback, we need to destroy and
1796 	 * recreate it. That creates a window of opportunity where RALUE and
1797 	 * RATR registers end up referencing a RIF that's already gone. RATRs
1798 	 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1799 	 * of RALUE, demote the decap route back.
1800 	 */
1801 	if (ipip_entry->decap_fib_entry)
1802 		mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1803 
1804 	if (recreate_loopback) {
1805 		err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1806 						       keep_encap, extack);
1807 		if (err)
1808 			return err;
1809 	} else if (update_nexthops) {
1810 		mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1811 					    &ipip_entry->ol_lb->common);
1812 	}
1813 
1814 	if (ipip_entry->ol_dev->flags & IFF_UP)
1815 		mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1816 
1817 	return 0;
1818 }
1819 
1820 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1821 						struct net_device *ol_dev,
1822 						struct netlink_ext_ack *extack)
1823 {
1824 	struct mlxsw_sp_ipip_entry *ipip_entry =
1825 		mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1826 
1827 	if (!ipip_entry)
1828 		return 0;
1829 
1830 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1831 						   true, false, false, extack);
1832 }
1833 
1834 static int
1835 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1836 				     struct mlxsw_sp_ipip_entry *ipip_entry,
1837 				     struct net_device *ul_dev,
1838 				     bool *demote_this,
1839 				     struct netlink_ext_ack *extack)
1840 {
1841 	u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1842 	enum mlxsw_sp_l3proto ul_proto;
1843 	union mlxsw_sp_l3addr saddr;
1844 
1845 	/* Moving underlay to a different VRF might cause local address
1846 	 * conflict, and the conflicting tunnels need to be demoted.
1847 	 */
1848 	ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1849 	saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1850 	if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1851 						 saddr, ul_tb_id,
1852 						 ipip_entry)) {
1853 		*demote_this = true;
1854 		return 0;
1855 	}
1856 
1857 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1858 						   true, true, false, extack);
1859 }
1860 
1861 static int
1862 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1863 				    struct mlxsw_sp_ipip_entry *ipip_entry,
1864 				    struct net_device *ul_dev)
1865 {
1866 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1867 						   false, false, true, NULL);
1868 }
1869 
1870 static int
1871 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1872 				      struct mlxsw_sp_ipip_entry *ipip_entry,
1873 				      struct net_device *ul_dev)
1874 {
1875 	/* A down underlay device causes encapsulated packets to not be
1876 	 * forwarded, but decap still works. So refresh next hops without
1877 	 * touching anything else.
1878 	 */
1879 	return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1880 						   false, false, true, NULL);
1881 }
1882 
1883 static int
1884 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1885 					struct net_device *ol_dev,
1886 					struct netlink_ext_ack *extack)
1887 {
1888 	const struct mlxsw_sp_ipip_ops *ipip_ops;
1889 	struct mlxsw_sp_ipip_entry *ipip_entry;
1890 	int err;
1891 
1892 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1893 	if (!ipip_entry)
1894 		/* A change might make a tunnel eligible for offloading, but
1895 		 * that is currently not implemented. What falls to slow path
1896 		 * stays there.
1897 		 */
1898 		return 0;
1899 
1900 	/* A change might make a tunnel not eligible for offloading. */
1901 	if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1902 						 ipip_entry->ipipt)) {
1903 		mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1904 		return 0;
1905 	}
1906 
1907 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1908 	err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1909 	return err;
1910 }
1911 
1912 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1913 				       struct mlxsw_sp_ipip_entry *ipip_entry)
1914 {
1915 	struct net_device *ol_dev = ipip_entry->ol_dev;
1916 
1917 	if (ol_dev->flags & IFF_UP)
1918 		mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1919 	mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1920 }
1921 
1922 /* The configuration where several tunnels have the same local address in the
1923  * same underlay table needs special treatment in the HW. That is currently not
1924  * implemented in the driver. This function finds and demotes the first tunnel
1925  * with a given source address, except the one passed in the argument
1926  * `except'.
1927  */
1928 bool
1929 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1930 				     enum mlxsw_sp_l3proto ul_proto,
1931 				     union mlxsw_sp_l3addr saddr,
1932 				     u32 ul_tb_id,
1933 				     const struct mlxsw_sp_ipip_entry *except)
1934 {
1935 	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1936 
1937 	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1938 				 ipip_list_node) {
1939 		if (ipip_entry != except &&
1940 		    mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1941 						      ul_tb_id, ipip_entry)) {
1942 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1943 			return true;
1944 		}
1945 	}
1946 
1947 	return false;
1948 }
1949 
1950 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1951 						     struct net_device *ul_dev)
1952 {
1953 	struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1954 
1955 	list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1956 				 ipip_list_node) {
1957 		struct net_device *ol_dev = ipip_entry->ol_dev;
1958 		struct net_device *ipip_ul_dev;
1959 
1960 		rcu_read_lock();
1961 		ipip_ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1962 		rcu_read_unlock();
1963 		if (ipip_ul_dev == ul_dev)
1964 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1965 	}
1966 }
1967 
1968 static int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1969 					    struct net_device *ol_dev,
1970 					    unsigned long event,
1971 					    struct netdev_notifier_info *info)
1972 {
1973 	struct netdev_notifier_changeupper_info *chup;
1974 	struct netlink_ext_ack *extack;
1975 	int err = 0;
1976 
1977 	switch (event) {
1978 	case NETDEV_REGISTER:
1979 		err = mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1980 		break;
1981 	case NETDEV_UNREGISTER:
1982 		mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1983 		break;
1984 	case NETDEV_UP:
1985 		mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1986 		break;
1987 	case NETDEV_DOWN:
1988 		mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1989 		break;
1990 	case NETDEV_CHANGEUPPER:
1991 		chup = container_of(info, typeof(*chup), info);
1992 		extack = info->extack;
1993 		if (netif_is_l3_master(chup->upper_dev))
1994 			err = mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1995 								   ol_dev,
1996 								   extack);
1997 		break;
1998 	case NETDEV_CHANGE:
1999 		extack = info->extack;
2000 		err = mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
2001 							      ol_dev, extack);
2002 		break;
2003 	case NETDEV_CHANGEMTU:
2004 		err = mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
2005 		break;
2006 	}
2007 	return err;
2008 }
2009 
2010 static int
2011 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2012 				   struct mlxsw_sp_ipip_entry *ipip_entry,
2013 				   struct net_device *ul_dev,
2014 				   bool *demote_this,
2015 				   unsigned long event,
2016 				   struct netdev_notifier_info *info)
2017 {
2018 	struct netdev_notifier_changeupper_info *chup;
2019 	struct netlink_ext_ack *extack;
2020 
2021 	switch (event) {
2022 	case NETDEV_CHANGEUPPER:
2023 		chup = container_of(info, typeof(*chup), info);
2024 		extack = info->extack;
2025 		if (netif_is_l3_master(chup->upper_dev))
2026 			return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
2027 								    ipip_entry,
2028 								    ul_dev,
2029 								    demote_this,
2030 								    extack);
2031 		break;
2032 
2033 	case NETDEV_UP:
2034 		return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
2035 							   ul_dev);
2036 	case NETDEV_DOWN:
2037 		return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
2038 							     ipip_entry,
2039 							     ul_dev);
2040 	}
2041 	return 0;
2042 }
2043 
2044 static int
2045 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
2046 				 struct net_device *ul_dev,
2047 				 unsigned long event,
2048 				 struct netdev_notifier_info *info)
2049 {
2050 	struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
2051 	int err;
2052 
2053 	while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
2054 								ul_dev,
2055 								ipip_entry))) {
2056 		struct mlxsw_sp_ipip_entry *prev;
2057 		bool demote_this = false;
2058 
2059 		err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
2060 							 ul_dev, &demote_this,
2061 							 event, info);
2062 		if (err) {
2063 			mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
2064 								 ul_dev);
2065 			return err;
2066 		}
2067 
2068 		if (demote_this) {
2069 			if (list_is_first(&ipip_entry->ipip_list_node,
2070 					  &mlxsw_sp->router->ipip_list))
2071 				prev = NULL;
2072 			else
2073 				/* This can't be cached from previous iteration,
2074 				 * because that entry could be gone now.
2075 				 */
2076 				prev = list_prev_entry(ipip_entry,
2077 						       ipip_list_node);
2078 			mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
2079 			ipip_entry = prev;
2080 		}
2081 	}
2082 
2083 	return 0;
2084 }
2085 
2086 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2087 				      enum mlxsw_sp_l3proto ul_proto,
2088 				      const union mlxsw_sp_l3addr *ul_sip,
2089 				      u32 tunnel_index)
2090 {
2091 	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2092 	struct mlxsw_sp_router *router = mlxsw_sp->router;
2093 	struct mlxsw_sp_fib_entry *fib_entry;
2094 	int err = 0;
2095 
2096 	mutex_lock(&mlxsw_sp->router->lock);
2097 
2098 	if (WARN_ON_ONCE(router->nve_decap_config.valid)) {
2099 		err = -EINVAL;
2100 		goto out;
2101 	}
2102 
2103 	router->nve_decap_config.ul_tb_id = ul_tb_id;
2104 	router->nve_decap_config.tunnel_index = tunnel_index;
2105 	router->nve_decap_config.ul_proto = ul_proto;
2106 	router->nve_decap_config.ul_sip = *ul_sip;
2107 	router->nve_decap_config.valid = true;
2108 
2109 	/* It is valid to create a tunnel with a local IP and only later
2110 	 * assign this IP address to a local interface
2111 	 */
2112 	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2113 							 ul_proto, ul_sip,
2114 							 type);
2115 	if (!fib_entry)
2116 		goto out;
2117 
2118 	fib_entry->decap.tunnel_index = tunnel_index;
2119 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2120 
2121 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2122 	if (err)
2123 		goto err_fib_entry_update;
2124 
2125 	goto out;
2126 
2127 err_fib_entry_update:
2128 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2129 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2130 out:
2131 	mutex_unlock(&mlxsw_sp->router->lock);
2132 	return err;
2133 }
2134 
2135 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
2136 				      enum mlxsw_sp_l3proto ul_proto,
2137 				      const union mlxsw_sp_l3addr *ul_sip)
2138 {
2139 	enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
2140 	struct mlxsw_sp_router *router = mlxsw_sp->router;
2141 	struct mlxsw_sp_fib_entry *fib_entry;
2142 
2143 	mutex_lock(&mlxsw_sp->router->lock);
2144 
2145 	if (WARN_ON_ONCE(!router->nve_decap_config.valid))
2146 		goto out;
2147 
2148 	router->nve_decap_config.valid = false;
2149 
2150 	fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
2151 							 ul_proto, ul_sip,
2152 							 type);
2153 	if (!fib_entry)
2154 		goto out;
2155 
2156 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2157 	mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2158 out:
2159 	mutex_unlock(&mlxsw_sp->router->lock);
2160 }
2161 
2162 static bool mlxsw_sp_router_nve_is_decap(struct mlxsw_sp *mlxsw_sp,
2163 					 u32 ul_tb_id,
2164 					 enum mlxsw_sp_l3proto ul_proto,
2165 					 const union mlxsw_sp_l3addr *ul_sip)
2166 {
2167 	struct mlxsw_sp_router *router = mlxsw_sp->router;
2168 
2169 	return router->nve_decap_config.valid &&
2170 	       router->nve_decap_config.ul_tb_id == ul_tb_id &&
2171 	       router->nve_decap_config.ul_proto == ul_proto &&
2172 	       !memcmp(&router->nve_decap_config.ul_sip, ul_sip,
2173 		       sizeof(*ul_sip));
2174 }
2175 
2176 struct mlxsw_sp_neigh_key {
2177 	struct neighbour *n;
2178 };
2179 
2180 struct mlxsw_sp_neigh_entry {
2181 	struct list_head rif_list_node;
2182 	struct rhash_head ht_node;
2183 	struct mlxsw_sp_neigh_key key;
2184 	u16 rif;
2185 	bool connected;
2186 	unsigned char ha[ETH_ALEN];
2187 	struct list_head nexthop_list; /* list of nexthops using
2188 					* this neigh entry
2189 					*/
2190 	struct list_head nexthop_neighs_list_node;
2191 	unsigned int counter_index;
2192 	bool counter_valid;
2193 };
2194 
2195 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
2196 	.key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
2197 	.head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
2198 	.key_len = sizeof(struct mlxsw_sp_neigh_key),
2199 };
2200 
2201 struct mlxsw_sp_neigh_entry *
2202 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
2203 			struct mlxsw_sp_neigh_entry *neigh_entry)
2204 {
2205 	if (!neigh_entry) {
2206 		if (list_empty(&rif->neigh_list))
2207 			return NULL;
2208 		else
2209 			return list_first_entry(&rif->neigh_list,
2210 						typeof(*neigh_entry),
2211 						rif_list_node);
2212 	}
2213 	if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
2214 		return NULL;
2215 	return list_next_entry(neigh_entry, rif_list_node);
2216 }
2217 
2218 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
2219 {
2220 	return neigh_entry->key.n->tbl->family;
2221 }
2222 
2223 unsigned char *
2224 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
2225 {
2226 	return neigh_entry->ha;
2227 }
2228 
2229 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2230 {
2231 	struct neighbour *n;
2232 
2233 	n = neigh_entry->key.n;
2234 	return ntohl(*((__be32 *) n->primary_key));
2235 }
2236 
2237 struct in6_addr *
2238 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
2239 {
2240 	struct neighbour *n;
2241 
2242 	n = neigh_entry->key.n;
2243 	return (struct in6_addr *) &n->primary_key;
2244 }
2245 
2246 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
2247 			       struct mlxsw_sp_neigh_entry *neigh_entry,
2248 			       u64 *p_counter)
2249 {
2250 	if (!neigh_entry->counter_valid)
2251 		return -EINVAL;
2252 
2253 	return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
2254 					 p_counter, NULL);
2255 }
2256 
2257 static struct mlxsw_sp_neigh_entry *
2258 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
2259 			   u16 rif)
2260 {
2261 	struct mlxsw_sp_neigh_entry *neigh_entry;
2262 
2263 	neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
2264 	if (!neigh_entry)
2265 		return NULL;
2266 
2267 	neigh_entry->key.n = n;
2268 	neigh_entry->rif = rif;
2269 	INIT_LIST_HEAD(&neigh_entry->nexthop_list);
2270 
2271 	return neigh_entry;
2272 }
2273 
2274 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
2275 {
2276 	kfree(neigh_entry);
2277 }
2278 
2279 static int
2280 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
2281 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2282 {
2283 	return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
2284 				      &neigh_entry->ht_node,
2285 				      mlxsw_sp_neigh_ht_params);
2286 }
2287 
2288 static void
2289 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
2290 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2291 {
2292 	rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2293 			       &neigh_entry->ht_node,
2294 			       mlxsw_sp_neigh_ht_params);
2295 }
2296 
2297 static bool
2298 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2299 				    struct mlxsw_sp_neigh_entry *neigh_entry)
2300 {
2301 	struct devlink *devlink;
2302 	const char *table_name;
2303 
2304 	switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2305 	case AF_INET:
2306 		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2307 		break;
2308 	case AF_INET6:
2309 		table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2310 		break;
2311 	default:
2312 		WARN_ON(1);
2313 		return false;
2314 	}
2315 
2316 	devlink = priv_to_devlink(mlxsw_sp->core);
2317 	return devlink_dpipe_table_counter_enabled(devlink, table_name);
2318 }
2319 
2320 static void
2321 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2322 			     struct mlxsw_sp_neigh_entry *neigh_entry)
2323 {
2324 	if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2325 		return;
2326 
2327 	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2328 		return;
2329 
2330 	neigh_entry->counter_valid = true;
2331 }
2332 
2333 static void
2334 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2335 			    struct mlxsw_sp_neigh_entry *neigh_entry)
2336 {
2337 	if (!neigh_entry->counter_valid)
2338 		return;
2339 	mlxsw_sp_flow_counter_free(mlxsw_sp,
2340 				   neigh_entry->counter_index);
2341 	neigh_entry->counter_valid = false;
2342 }
2343 
2344 static struct mlxsw_sp_neigh_entry *
2345 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2346 {
2347 	struct mlxsw_sp_neigh_entry *neigh_entry;
2348 	struct mlxsw_sp_rif *rif;
2349 	int err;
2350 
2351 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2352 	if (!rif)
2353 		return ERR_PTR(-EINVAL);
2354 
2355 	neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2356 	if (!neigh_entry)
2357 		return ERR_PTR(-ENOMEM);
2358 
2359 	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2360 	if (err)
2361 		goto err_neigh_entry_insert;
2362 
2363 	mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2364 	atomic_inc(&mlxsw_sp->router->neighs_update.neigh_count);
2365 	list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2366 
2367 	return neigh_entry;
2368 
2369 err_neigh_entry_insert:
2370 	mlxsw_sp_neigh_entry_free(neigh_entry);
2371 	return ERR_PTR(err);
2372 }
2373 
2374 static void
2375 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2376 			     struct mlxsw_sp_neigh_entry *neigh_entry)
2377 {
2378 	list_del(&neigh_entry->rif_list_node);
2379 	atomic_dec(&mlxsw_sp->router->neighs_update.neigh_count);
2380 	mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2381 	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2382 	mlxsw_sp_neigh_entry_free(neigh_entry);
2383 }
2384 
2385 static struct mlxsw_sp_neigh_entry *
2386 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2387 {
2388 	struct mlxsw_sp_neigh_key key;
2389 
2390 	key.n = n;
2391 	return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2392 				      &key, mlxsw_sp_neigh_ht_params);
2393 }
2394 
2395 static void
2396 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2397 {
2398 	unsigned long interval;
2399 
2400 #if IS_ENABLED(CONFIG_IPV6)
2401 	interval = min_t(unsigned long,
2402 			 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2403 			 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2404 #else
2405 	interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2406 #endif
2407 	mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2408 }
2409 
2410 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2411 						   char *rauhtd_pl,
2412 						   int ent_index)
2413 {
2414 	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
2415 	struct net_device *dev;
2416 	struct neighbour *n;
2417 	__be32 dipn;
2418 	u32 dip;
2419 	u16 rif;
2420 
2421 	mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2422 
2423 	if (WARN_ON_ONCE(rif >= max_rifs))
2424 		return;
2425 	if (!mlxsw_sp->router->rifs[rif]) {
2426 		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2427 		return;
2428 	}
2429 
2430 	dipn = htonl(dip);
2431 	dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
2432 	n = neigh_lookup(&arp_tbl, &dipn, dev);
2433 	if (!n)
2434 		return;
2435 
2436 	netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2437 	neigh_event_send(n, NULL);
2438 	neigh_release(n);
2439 }
2440 
2441 #if IS_ENABLED(CONFIG_IPV6)
2442 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2443 						   char *rauhtd_pl,
2444 						   int rec_index)
2445 {
2446 	struct net_device *dev;
2447 	struct neighbour *n;
2448 	struct in6_addr dip;
2449 	u16 rif;
2450 
2451 	mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2452 					 (char *) &dip);
2453 
2454 	if (!mlxsw_sp->router->rifs[rif]) {
2455 		dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2456 		return;
2457 	}
2458 
2459 	dev = mlxsw_sp_rif_dev(mlxsw_sp->router->rifs[rif]);
2460 	n = neigh_lookup(&nd_tbl, &dip, dev);
2461 	if (!n)
2462 		return;
2463 
2464 	netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2465 	neigh_event_send(n, NULL);
2466 	neigh_release(n);
2467 }
2468 #else
2469 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2470 						   char *rauhtd_pl,
2471 						   int rec_index)
2472 {
2473 }
2474 #endif
2475 
2476 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2477 						   char *rauhtd_pl,
2478 						   int rec_index)
2479 {
2480 	u8 num_entries;
2481 	int i;
2482 
2483 	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2484 								rec_index);
2485 	/* Hardware starts counting at 0, so add 1. */
2486 	num_entries++;
2487 
2488 	/* Each record consists of several neighbour entries. */
2489 	for (i = 0; i < num_entries; i++) {
2490 		int ent_index;
2491 
2492 		ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2493 		mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2494 						       ent_index);
2495 	}
2496 
2497 }
2498 
2499 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2500 						   char *rauhtd_pl,
2501 						   int rec_index)
2502 {
2503 	/* One record contains one entry. */
2504 	mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2505 					       rec_index);
2506 }
2507 
2508 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2509 					      char *rauhtd_pl, int rec_index)
2510 {
2511 	switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2512 	case MLXSW_REG_RAUHTD_TYPE_IPV4:
2513 		mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2514 						       rec_index);
2515 		break;
2516 	case MLXSW_REG_RAUHTD_TYPE_IPV6:
2517 		mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2518 						       rec_index);
2519 		break;
2520 	}
2521 }
2522 
2523 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2524 {
2525 	u8 num_rec, last_rec_index, num_entries;
2526 
2527 	num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2528 	last_rec_index = num_rec - 1;
2529 
2530 	if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2531 		return false;
2532 	if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2533 	    MLXSW_REG_RAUHTD_TYPE_IPV6)
2534 		return true;
2535 
2536 	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2537 								last_rec_index);
2538 	if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2539 		return true;
2540 	return false;
2541 }
2542 
2543 static int
2544 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2545 				       char *rauhtd_pl,
2546 				       enum mlxsw_reg_rauhtd_type type)
2547 {
2548 	int i, num_rec;
2549 	int err;
2550 
2551 	/* Ensure the RIF we read from the device does not change mid-dump. */
2552 	mutex_lock(&mlxsw_sp->router->lock);
2553 	do {
2554 		mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2555 		err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2556 				      rauhtd_pl);
2557 		if (err) {
2558 			dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2559 			break;
2560 		}
2561 		num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2562 		for (i = 0; i < num_rec; i++)
2563 			mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2564 							  i);
2565 	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2566 	mutex_unlock(&mlxsw_sp->router->lock);
2567 
2568 	return err;
2569 }
2570 
2571 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2572 {
2573 	enum mlxsw_reg_rauhtd_type type;
2574 	char *rauhtd_pl;
2575 	int err;
2576 
2577 	if (!atomic_read(&mlxsw_sp->router->neighs_update.neigh_count))
2578 		return 0;
2579 
2580 	rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2581 	if (!rauhtd_pl)
2582 		return -ENOMEM;
2583 
2584 	type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2585 	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2586 	if (err)
2587 		goto out;
2588 
2589 	type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2590 	err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2591 out:
2592 	kfree(rauhtd_pl);
2593 	return err;
2594 }
2595 
2596 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2597 {
2598 	struct mlxsw_sp_neigh_entry *neigh_entry;
2599 
2600 	mutex_lock(&mlxsw_sp->router->lock);
2601 	list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2602 			    nexthop_neighs_list_node)
2603 		/* If this neigh have nexthops, make the kernel think this neigh
2604 		 * is active regardless of the traffic.
2605 		 */
2606 		neigh_event_send(neigh_entry->key.n, NULL);
2607 	mutex_unlock(&mlxsw_sp->router->lock);
2608 }
2609 
2610 static void
2611 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2612 {
2613 	unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2614 
2615 	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2616 			       msecs_to_jiffies(interval));
2617 }
2618 
2619 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2620 {
2621 	struct mlxsw_sp_router *router;
2622 	int err;
2623 
2624 	router = container_of(work, struct mlxsw_sp_router,
2625 			      neighs_update.dw.work);
2626 	err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2627 	if (err)
2628 		dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2629 
2630 	mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2631 
2632 	mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2633 }
2634 
2635 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2636 {
2637 	struct mlxsw_sp_neigh_entry *neigh_entry;
2638 	struct mlxsw_sp_router *router;
2639 
2640 	router = container_of(work, struct mlxsw_sp_router,
2641 			      nexthop_probe_dw.work);
2642 	/* Iterate over nexthop neighbours, find those who are unresolved and
2643 	 * send arp on them. This solves the chicken-egg problem when
2644 	 * the nexthop wouldn't get offloaded until the neighbor is resolved
2645 	 * but it wouldn't get resolved ever in case traffic is flowing in HW
2646 	 * using different nexthop.
2647 	 */
2648 	mutex_lock(&router->lock);
2649 	list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2650 			    nexthop_neighs_list_node)
2651 		if (!neigh_entry->connected)
2652 			neigh_event_send(neigh_entry->key.n, NULL);
2653 	mutex_unlock(&router->lock);
2654 
2655 	mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2656 			       MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2657 }
2658 
2659 static void
2660 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2661 			      struct mlxsw_sp_neigh_entry *neigh_entry,
2662 			      bool removing, bool dead);
2663 
2664 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2665 {
2666 	return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2667 			MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2668 }
2669 
2670 static int
2671 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2672 				struct mlxsw_sp_neigh_entry *neigh_entry,
2673 				enum mlxsw_reg_rauht_op op)
2674 {
2675 	struct neighbour *n = neigh_entry->key.n;
2676 	u32 dip = ntohl(*((__be32 *) n->primary_key));
2677 	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2678 
2679 	mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2680 			      dip);
2681 	if (neigh_entry->counter_valid)
2682 		mlxsw_reg_rauht_pack_counter(rauht_pl,
2683 					     neigh_entry->counter_index);
2684 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2685 }
2686 
2687 static int
2688 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2689 				struct mlxsw_sp_neigh_entry *neigh_entry,
2690 				enum mlxsw_reg_rauht_op op)
2691 {
2692 	struct neighbour *n = neigh_entry->key.n;
2693 	char rauht_pl[MLXSW_REG_RAUHT_LEN];
2694 	const char *dip = n->primary_key;
2695 
2696 	mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2697 			      dip);
2698 	if (neigh_entry->counter_valid)
2699 		mlxsw_reg_rauht_pack_counter(rauht_pl,
2700 					     neigh_entry->counter_index);
2701 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2702 }
2703 
2704 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2705 {
2706 	struct neighbour *n = neigh_entry->key.n;
2707 
2708 	/* Packets with a link-local destination address are trapped
2709 	 * after LPM lookup and never reach the neighbour table, so
2710 	 * there is no need to program such neighbours to the device.
2711 	 */
2712 	if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2713 	    IPV6_ADDR_LINKLOCAL)
2714 		return true;
2715 	return false;
2716 }
2717 
2718 static void
2719 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2720 			    struct mlxsw_sp_neigh_entry *neigh_entry,
2721 			    bool adding)
2722 {
2723 	enum mlxsw_reg_rauht_op op = mlxsw_sp_rauht_op(adding);
2724 	int err;
2725 
2726 	if (!adding && !neigh_entry->connected)
2727 		return;
2728 	neigh_entry->connected = adding;
2729 	if (neigh_entry->key.n->tbl->family == AF_INET) {
2730 		err = mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2731 						      op);
2732 		if (err)
2733 			return;
2734 	} else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2735 		if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2736 			return;
2737 		err = mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2738 						      op);
2739 		if (err)
2740 			return;
2741 	} else {
2742 		WARN_ON_ONCE(1);
2743 		return;
2744 	}
2745 
2746 	if (adding)
2747 		neigh_entry->key.n->flags |= NTF_OFFLOADED;
2748 	else
2749 		neigh_entry->key.n->flags &= ~NTF_OFFLOADED;
2750 }
2751 
2752 void
2753 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2754 				    struct mlxsw_sp_neigh_entry *neigh_entry,
2755 				    bool adding)
2756 {
2757 	if (adding)
2758 		mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2759 	else
2760 		mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2761 	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2762 }
2763 
2764 struct mlxsw_sp_netevent_work {
2765 	struct work_struct work;
2766 	struct mlxsw_sp *mlxsw_sp;
2767 	struct neighbour *n;
2768 };
2769 
2770 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2771 {
2772 	struct mlxsw_sp_netevent_work *net_work =
2773 		container_of(work, struct mlxsw_sp_netevent_work, work);
2774 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2775 	struct mlxsw_sp_neigh_entry *neigh_entry;
2776 	struct neighbour *n = net_work->n;
2777 	unsigned char ha[ETH_ALEN];
2778 	bool entry_connected;
2779 	u8 nud_state, dead;
2780 
2781 	/* If these parameters are changed after we release the lock,
2782 	 * then we are guaranteed to receive another event letting us
2783 	 * know about it.
2784 	 */
2785 	read_lock_bh(&n->lock);
2786 	memcpy(ha, n->ha, ETH_ALEN);
2787 	nud_state = n->nud_state;
2788 	dead = n->dead;
2789 	read_unlock_bh(&n->lock);
2790 
2791 	mutex_lock(&mlxsw_sp->router->lock);
2792 	mlxsw_sp_span_respin(mlxsw_sp);
2793 
2794 	entry_connected = nud_state & NUD_VALID && !dead;
2795 	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2796 	if (!entry_connected && !neigh_entry)
2797 		goto out;
2798 	if (!neigh_entry) {
2799 		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2800 		if (IS_ERR(neigh_entry))
2801 			goto out;
2802 	}
2803 
2804 	if (neigh_entry->connected && entry_connected &&
2805 	    !memcmp(neigh_entry->ha, ha, ETH_ALEN))
2806 		goto out;
2807 
2808 	memcpy(neigh_entry->ha, ha, ETH_ALEN);
2809 	mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2810 	mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected,
2811 				      dead);
2812 
2813 	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2814 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2815 
2816 out:
2817 	mutex_unlock(&mlxsw_sp->router->lock);
2818 	neigh_release(n);
2819 	kfree(net_work);
2820 }
2821 
2822 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2823 
2824 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2825 {
2826 	struct mlxsw_sp_netevent_work *net_work =
2827 		container_of(work, struct mlxsw_sp_netevent_work, work);
2828 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2829 
2830 	mlxsw_sp_mp_hash_init(mlxsw_sp);
2831 	kfree(net_work);
2832 }
2833 
2834 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2835 
2836 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2837 {
2838 	struct mlxsw_sp_netevent_work *net_work =
2839 		container_of(work, struct mlxsw_sp_netevent_work, work);
2840 	struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2841 
2842 	__mlxsw_sp_router_init(mlxsw_sp);
2843 	kfree(net_work);
2844 }
2845 
2846 static int mlxsw_sp_router_schedule_work(struct net *net,
2847 					 struct mlxsw_sp_router *router,
2848 					 struct neighbour *n,
2849 					 void (*cb)(struct work_struct *))
2850 {
2851 	struct mlxsw_sp_netevent_work *net_work;
2852 
2853 	if (!net_eq(net, mlxsw_sp_net(router->mlxsw_sp)))
2854 		return NOTIFY_DONE;
2855 
2856 	net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2857 	if (!net_work)
2858 		return NOTIFY_BAD;
2859 
2860 	INIT_WORK(&net_work->work, cb);
2861 	net_work->mlxsw_sp = router->mlxsw_sp;
2862 	net_work->n = n;
2863 	mlxsw_core_schedule_work(&net_work->work);
2864 	return NOTIFY_DONE;
2865 }
2866 
2867 static bool mlxsw_sp_dev_lower_is_port(struct net_device *dev)
2868 {
2869 	struct mlxsw_sp_port *mlxsw_sp_port;
2870 
2871 	rcu_read_lock();
2872 	mlxsw_sp_port = mlxsw_sp_port_dev_lower_find_rcu(dev);
2873 	rcu_read_unlock();
2874 	return !!mlxsw_sp_port;
2875 }
2876 
2877 static int mlxsw_sp_router_schedule_neigh_work(struct mlxsw_sp_router *router,
2878 					       struct neighbour *n)
2879 {
2880 	struct net *net;
2881 
2882 	net = neigh_parms_net(n->parms);
2883 
2884 	/* Take a reference to ensure the neighbour won't be destructed until we
2885 	 * drop the reference in delayed work.
2886 	 */
2887 	neigh_clone(n);
2888 	return mlxsw_sp_router_schedule_work(net, router, n,
2889 					     mlxsw_sp_router_neigh_event_work);
2890 }
2891 
2892 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2893 					  unsigned long event, void *ptr)
2894 {
2895 	struct mlxsw_sp_router *router;
2896 	unsigned long interval;
2897 	struct neigh_parms *p;
2898 	struct neighbour *n;
2899 
2900 	router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2901 
2902 	switch (event) {
2903 	case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2904 		p = ptr;
2905 
2906 		/* We don't care about changes in the default table. */
2907 		if (!p->dev || (p->tbl->family != AF_INET &&
2908 				p->tbl->family != AF_INET6))
2909 			return NOTIFY_DONE;
2910 
2911 		/* We are in atomic context and can't take RTNL mutex,
2912 		 * so use RCU variant to walk the device chain.
2913 		 */
2914 		if (!mlxsw_sp_dev_lower_is_port(p->dev))
2915 			return NOTIFY_DONE;
2916 
2917 		interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2918 		router->neighs_update.interval = interval;
2919 		break;
2920 	case NETEVENT_NEIGH_UPDATE:
2921 		n = ptr;
2922 
2923 		if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2924 			return NOTIFY_DONE;
2925 
2926 		if (!mlxsw_sp_dev_lower_is_port(n->dev))
2927 			return NOTIFY_DONE;
2928 
2929 		return mlxsw_sp_router_schedule_neigh_work(router, n);
2930 
2931 	case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2932 	case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2933 		return mlxsw_sp_router_schedule_work(ptr, router, NULL,
2934 				mlxsw_sp_router_mp_hash_event_work);
2935 
2936 	case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2937 		return mlxsw_sp_router_schedule_work(ptr, router, NULL,
2938 				mlxsw_sp_router_update_priority_work);
2939 	}
2940 
2941 	return NOTIFY_DONE;
2942 }
2943 
2944 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2945 {
2946 	int err;
2947 
2948 	err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2949 			      &mlxsw_sp_neigh_ht_params);
2950 	if (err)
2951 		return err;
2952 
2953 	/* Initialize the polling interval according to the default
2954 	 * table.
2955 	 */
2956 	mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2957 
2958 	/* Create the delayed works for the activity_update */
2959 	INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2960 			  mlxsw_sp_router_neighs_update_work);
2961 	INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2962 			  mlxsw_sp_router_probe_unresolved_nexthops);
2963 	atomic_set(&mlxsw_sp->router->neighs_update.neigh_count, 0);
2964 	mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2965 	mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2966 	return 0;
2967 }
2968 
2969 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2970 {
2971 	cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2972 	cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2973 	rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2974 }
2975 
2976 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2977 					 struct mlxsw_sp_rif *rif)
2978 {
2979 	struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2980 
2981 	list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2982 				 rif_list_node) {
2983 		mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2984 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2985 	}
2986 }
2987 
2988 struct mlxsw_sp_neigh_rif_made_sync {
2989 	struct mlxsw_sp *mlxsw_sp;
2990 	struct mlxsw_sp_rif *rif;
2991 	int err;
2992 };
2993 
2994 static void mlxsw_sp_neigh_rif_made_sync_each(struct neighbour *n, void *data)
2995 {
2996 	struct mlxsw_sp_neigh_rif_made_sync *rms = data;
2997 	int rc;
2998 
2999 	if (rms->err)
3000 		return;
3001 	if (n->dev != mlxsw_sp_rif_dev(rms->rif))
3002 		return;
3003 	rc = mlxsw_sp_router_schedule_neigh_work(rms->mlxsw_sp->router, n);
3004 	if (rc != NOTIFY_DONE)
3005 		rms->err = -ENOMEM;
3006 }
3007 
3008 static int mlxsw_sp_neigh_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
3009 					struct mlxsw_sp_rif *rif)
3010 {
3011 	struct mlxsw_sp_neigh_rif_made_sync rms = {
3012 		.mlxsw_sp = mlxsw_sp,
3013 		.rif = rif,
3014 	};
3015 
3016 	neigh_for_each(&arp_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
3017 	if (rms.err)
3018 		goto err_arp;
3019 
3020 #if IS_ENABLED(CONFIG_IPV6)
3021 	neigh_for_each(&nd_tbl, mlxsw_sp_neigh_rif_made_sync_each, &rms);
3022 #endif
3023 	if (rms.err)
3024 		goto err_nd;
3025 
3026 	return 0;
3027 
3028 err_nd:
3029 err_arp:
3030 	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
3031 	return rms.err;
3032 }
3033 
3034 enum mlxsw_sp_nexthop_type {
3035 	MLXSW_SP_NEXTHOP_TYPE_ETH,
3036 	MLXSW_SP_NEXTHOP_TYPE_IPIP,
3037 };
3038 
3039 enum mlxsw_sp_nexthop_action {
3040 	/* Nexthop forwards packets to an egress RIF */
3041 	MLXSW_SP_NEXTHOP_ACTION_FORWARD,
3042 	/* Nexthop discards packets */
3043 	MLXSW_SP_NEXTHOP_ACTION_DISCARD,
3044 	/* Nexthop traps packets */
3045 	MLXSW_SP_NEXTHOP_ACTION_TRAP,
3046 };
3047 
3048 struct mlxsw_sp_nexthop_key {
3049 	struct fib_nh *fib_nh;
3050 };
3051 
3052 struct mlxsw_sp_nexthop {
3053 	struct list_head neigh_list_node; /* member of neigh entry list */
3054 	struct list_head crif_list_node;
3055 	struct list_head router_list_node;
3056 	struct mlxsw_sp_nexthop_group_info *nhgi; /* pointer back to the group
3057 						   * this nexthop belongs to
3058 						   */
3059 	struct rhash_head ht_node;
3060 	struct neigh_table *neigh_tbl;
3061 	struct mlxsw_sp_nexthop_key key;
3062 	unsigned char gw_addr[sizeof(struct in6_addr)];
3063 	int ifindex;
3064 	int nh_weight;
3065 	int norm_nh_weight;
3066 	int num_adj_entries;
3067 	struct mlxsw_sp_crif *crif;
3068 	u8 should_offload:1, /* set indicates this nexthop should be written
3069 			      * to the adjacency table.
3070 			      */
3071 	   offloaded:1, /* set indicates this nexthop was written to the
3072 			 * adjacency table.
3073 			 */
3074 	   update:1; /* set indicates this nexthop should be updated in the
3075 		      * adjacency table (f.e., its MAC changed).
3076 		      */
3077 	enum mlxsw_sp_nexthop_action action;
3078 	enum mlxsw_sp_nexthop_type type;
3079 	union {
3080 		struct mlxsw_sp_neigh_entry *neigh_entry;
3081 		struct mlxsw_sp_ipip_entry *ipip_entry;
3082 	};
3083 	unsigned int counter_index;
3084 	bool counter_valid;
3085 };
3086 
3087 static struct net_device *
3088 mlxsw_sp_nexthop_dev(const struct mlxsw_sp_nexthop *nh)
3089 {
3090 	if (!nh->crif)
3091 		return NULL;
3092 	return nh->crif->key.dev;
3093 }
3094 
3095 enum mlxsw_sp_nexthop_group_type {
3096 	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4,
3097 	MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6,
3098 	MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ,
3099 };
3100 
3101 struct mlxsw_sp_nexthop_group_info {
3102 	struct mlxsw_sp_nexthop_group *nh_grp;
3103 	u32 adj_index;
3104 	u16 ecmp_size;
3105 	u16 count;
3106 	int sum_norm_weight;
3107 	u8 adj_index_valid:1,
3108 	   gateway:1, /* routes using the group use a gateway */
3109 	   is_resilient:1;
3110 	struct list_head list; /* member in nh_res_grp_list */
3111 	struct mlxsw_sp_nexthop nexthops[];
3112 };
3113 
3114 static struct mlxsw_sp_rif *
3115 mlxsw_sp_nhgi_rif(const struct mlxsw_sp_nexthop_group_info *nhgi)
3116 {
3117 	struct mlxsw_sp_crif *crif = nhgi->nexthops[0].crif;
3118 
3119 	if (!crif)
3120 		return NULL;
3121 	return crif->rif;
3122 }
3123 
3124 struct mlxsw_sp_nexthop_group_vr_key {
3125 	u16 vr_id;
3126 	enum mlxsw_sp_l3proto proto;
3127 };
3128 
3129 struct mlxsw_sp_nexthop_group_vr_entry {
3130 	struct list_head list; /* member in vr_list */
3131 	struct rhash_head ht_node; /* member in vr_ht */
3132 	refcount_t ref_count;
3133 	struct mlxsw_sp_nexthop_group_vr_key key;
3134 };
3135 
3136 struct mlxsw_sp_nexthop_group {
3137 	struct rhash_head ht_node;
3138 	struct list_head fib_list; /* list of fib entries that use this group */
3139 	union {
3140 		struct {
3141 			struct fib_info *fi;
3142 		} ipv4;
3143 		struct {
3144 			u32 id;
3145 		} obj;
3146 	};
3147 	struct mlxsw_sp_nexthop_group_info *nhgi;
3148 	struct list_head vr_list;
3149 	struct rhashtable vr_ht;
3150 	enum mlxsw_sp_nexthop_group_type type;
3151 	bool can_destroy;
3152 };
3153 
3154 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
3155 				    struct mlxsw_sp_nexthop *nh)
3156 {
3157 	struct devlink *devlink;
3158 
3159 	devlink = priv_to_devlink(mlxsw_sp->core);
3160 	if (!devlink_dpipe_table_counter_enabled(devlink,
3161 						 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
3162 		return;
3163 
3164 	if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
3165 		return;
3166 
3167 	nh->counter_valid = true;
3168 }
3169 
3170 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
3171 				   struct mlxsw_sp_nexthop *nh)
3172 {
3173 	if (!nh->counter_valid)
3174 		return;
3175 	mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
3176 	nh->counter_valid = false;
3177 }
3178 
3179 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
3180 				 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
3181 {
3182 	if (!nh->counter_valid)
3183 		return -EINVAL;
3184 
3185 	return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
3186 					 p_counter, NULL);
3187 }
3188 
3189 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
3190 					       struct mlxsw_sp_nexthop *nh)
3191 {
3192 	if (!nh) {
3193 		if (list_empty(&router->nexthop_list))
3194 			return NULL;
3195 		else
3196 			return list_first_entry(&router->nexthop_list,
3197 						typeof(*nh), router_list_node);
3198 	}
3199 	if (list_is_last(&nh->router_list_node, &router->nexthop_list))
3200 		return NULL;
3201 	return list_next_entry(nh, router_list_node);
3202 }
3203 
3204 bool mlxsw_sp_nexthop_is_forward(const struct mlxsw_sp_nexthop *nh)
3205 {
3206 	return nh->offloaded && nh->action == MLXSW_SP_NEXTHOP_ACTION_FORWARD;
3207 }
3208 
3209 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
3210 {
3211 	if (nh->type != MLXSW_SP_NEXTHOP_TYPE_ETH ||
3212 	    !mlxsw_sp_nexthop_is_forward(nh))
3213 		return NULL;
3214 	return nh->neigh_entry->ha;
3215 }
3216 
3217 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
3218 			     u32 *p_adj_size, u32 *p_adj_hash_index)
3219 {
3220 	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3221 	u32 adj_hash_index = 0;
3222 	int i;
3223 
3224 	if (!nh->offloaded || !nhgi->adj_index_valid)
3225 		return -EINVAL;
3226 
3227 	*p_adj_index = nhgi->adj_index;
3228 	*p_adj_size = nhgi->ecmp_size;
3229 
3230 	for (i = 0; i < nhgi->count; i++) {
3231 		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3232 
3233 		if (nh_iter == nh)
3234 			break;
3235 		if (nh_iter->offloaded)
3236 			adj_hash_index += nh_iter->num_adj_entries;
3237 	}
3238 
3239 	*p_adj_hash_index = adj_hash_index;
3240 	return 0;
3241 }
3242 
3243 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
3244 {
3245 	if (WARN_ON(!nh->crif))
3246 		return NULL;
3247 	return nh->crif->rif;
3248 }
3249 
3250 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
3251 {
3252 	struct mlxsw_sp_nexthop_group_info *nhgi = nh->nhgi;
3253 	int i;
3254 
3255 	for (i = 0; i < nhgi->count; i++) {
3256 		struct mlxsw_sp_nexthop *nh_iter = &nhgi->nexthops[i];
3257 
3258 		if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
3259 			return true;
3260 	}
3261 	return false;
3262 }
3263 
3264 static const struct rhashtable_params mlxsw_sp_nexthop_group_vr_ht_params = {
3265 	.key_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, key),
3266 	.head_offset = offsetof(struct mlxsw_sp_nexthop_group_vr_entry, ht_node),
3267 	.key_len = sizeof(struct mlxsw_sp_nexthop_group_vr_key),
3268 	.automatic_shrinking = true,
3269 };
3270 
3271 static struct mlxsw_sp_nexthop_group_vr_entry *
3272 mlxsw_sp_nexthop_group_vr_entry_lookup(struct mlxsw_sp_nexthop_group *nh_grp,
3273 				       const struct mlxsw_sp_fib *fib)
3274 {
3275 	struct mlxsw_sp_nexthop_group_vr_key key;
3276 
3277 	memset(&key, 0, sizeof(key));
3278 	key.vr_id = fib->vr->id;
3279 	key.proto = fib->proto;
3280 	return rhashtable_lookup_fast(&nh_grp->vr_ht, &key,
3281 				      mlxsw_sp_nexthop_group_vr_ht_params);
3282 }
3283 
3284 static int
3285 mlxsw_sp_nexthop_group_vr_entry_create(struct mlxsw_sp_nexthop_group *nh_grp,
3286 				       const struct mlxsw_sp_fib *fib)
3287 {
3288 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3289 	int err;
3290 
3291 	vr_entry = kzalloc(sizeof(*vr_entry), GFP_KERNEL);
3292 	if (!vr_entry)
3293 		return -ENOMEM;
3294 
3295 	vr_entry->key.vr_id = fib->vr->id;
3296 	vr_entry->key.proto = fib->proto;
3297 	refcount_set(&vr_entry->ref_count, 1);
3298 
3299 	err = rhashtable_insert_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3300 				     mlxsw_sp_nexthop_group_vr_ht_params);
3301 	if (err)
3302 		goto err_hashtable_insert;
3303 
3304 	list_add(&vr_entry->list, &nh_grp->vr_list);
3305 
3306 	return 0;
3307 
3308 err_hashtable_insert:
3309 	kfree(vr_entry);
3310 	return err;
3311 }
3312 
3313 static void
3314 mlxsw_sp_nexthop_group_vr_entry_destroy(struct mlxsw_sp_nexthop_group *nh_grp,
3315 					struct mlxsw_sp_nexthop_group_vr_entry *vr_entry)
3316 {
3317 	list_del(&vr_entry->list);
3318 	rhashtable_remove_fast(&nh_grp->vr_ht, &vr_entry->ht_node,
3319 			       mlxsw_sp_nexthop_group_vr_ht_params);
3320 	kfree(vr_entry);
3321 }
3322 
3323 static int
3324 mlxsw_sp_nexthop_group_vr_link(struct mlxsw_sp_nexthop_group *nh_grp,
3325 			       const struct mlxsw_sp_fib *fib)
3326 {
3327 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3328 
3329 	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3330 	if (vr_entry) {
3331 		refcount_inc(&vr_entry->ref_count);
3332 		return 0;
3333 	}
3334 
3335 	return mlxsw_sp_nexthop_group_vr_entry_create(nh_grp, fib);
3336 }
3337 
3338 static void
3339 mlxsw_sp_nexthop_group_vr_unlink(struct mlxsw_sp_nexthop_group *nh_grp,
3340 				 const struct mlxsw_sp_fib *fib)
3341 {
3342 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3343 
3344 	vr_entry = mlxsw_sp_nexthop_group_vr_entry_lookup(nh_grp, fib);
3345 	if (WARN_ON_ONCE(!vr_entry))
3346 		return;
3347 
3348 	if (!refcount_dec_and_test(&vr_entry->ref_count))
3349 		return;
3350 
3351 	mlxsw_sp_nexthop_group_vr_entry_destroy(nh_grp, vr_entry);
3352 }
3353 
3354 struct mlxsw_sp_nexthop_group_cmp_arg {
3355 	enum mlxsw_sp_nexthop_group_type type;
3356 	union {
3357 		struct fib_info *fi;
3358 		struct mlxsw_sp_fib6_entry *fib6_entry;
3359 		u32 id;
3360 	};
3361 };
3362 
3363 static bool
3364 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
3365 				    const struct in6_addr *gw, int ifindex,
3366 				    int weight)
3367 {
3368 	int i;
3369 
3370 	for (i = 0; i < nh_grp->nhgi->count; i++) {
3371 		const struct mlxsw_sp_nexthop *nh;
3372 
3373 		nh = &nh_grp->nhgi->nexthops[i];
3374 		if (nh->ifindex == ifindex && nh->nh_weight == weight &&
3375 		    ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
3376 			return true;
3377 	}
3378 
3379 	return false;
3380 }
3381 
3382 static bool
3383 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
3384 			    const struct mlxsw_sp_fib6_entry *fib6_entry)
3385 {
3386 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3387 
3388 	if (nh_grp->nhgi->count != fib6_entry->nrt6)
3389 		return false;
3390 
3391 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3392 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3393 		struct in6_addr *gw;
3394 		int ifindex, weight;
3395 
3396 		ifindex = fib6_nh->fib_nh_dev->ifindex;
3397 		weight = fib6_nh->fib_nh_weight;
3398 		gw = &fib6_nh->fib_nh_gw6;
3399 		if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
3400 							 weight))
3401 			return false;
3402 	}
3403 
3404 	return true;
3405 }
3406 
3407 static int
3408 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
3409 {
3410 	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
3411 	const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
3412 
3413 	if (nh_grp->type != cmp_arg->type)
3414 		return 1;
3415 
3416 	switch (cmp_arg->type) {
3417 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3418 		return cmp_arg->fi != nh_grp->ipv4.fi;
3419 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3420 		return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
3421 						    cmp_arg->fib6_entry);
3422 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3423 		return cmp_arg->id != nh_grp->obj.id;
3424 	default:
3425 		WARN_ON(1);
3426 		return 1;
3427 	}
3428 }
3429 
3430 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
3431 {
3432 	const struct mlxsw_sp_nexthop_group *nh_grp = data;
3433 	const struct mlxsw_sp_nexthop *nh;
3434 	struct fib_info *fi;
3435 	unsigned int val;
3436 	int i;
3437 
3438 	switch (nh_grp->type) {
3439 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3440 		fi = nh_grp->ipv4.fi;
3441 		return jhash(&fi, sizeof(fi), seed);
3442 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3443 		val = nh_grp->nhgi->count;
3444 		for (i = 0; i < nh_grp->nhgi->count; i++) {
3445 			nh = &nh_grp->nhgi->nexthops[i];
3446 			val ^= jhash(&nh->ifindex, sizeof(nh->ifindex), seed);
3447 			val ^= jhash(&nh->gw_addr, sizeof(nh->gw_addr), seed);
3448 		}
3449 		return jhash(&val, sizeof(val), seed);
3450 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3451 		return jhash(&nh_grp->obj.id, sizeof(nh_grp->obj.id), seed);
3452 	default:
3453 		WARN_ON(1);
3454 		return 0;
3455 	}
3456 }
3457 
3458 static u32
3459 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
3460 {
3461 	unsigned int val = fib6_entry->nrt6;
3462 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3463 
3464 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3465 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3466 		struct net_device *dev = fib6_nh->fib_nh_dev;
3467 		struct in6_addr *gw = &fib6_nh->fib_nh_gw6;
3468 
3469 		val ^= jhash(&dev->ifindex, sizeof(dev->ifindex), seed);
3470 		val ^= jhash(gw, sizeof(*gw), seed);
3471 	}
3472 
3473 	return jhash(&val, sizeof(val), seed);
3474 }
3475 
3476 static u32
3477 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
3478 {
3479 	const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
3480 
3481 	switch (cmp_arg->type) {
3482 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
3483 		return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
3484 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
3485 		return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
3486 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
3487 		return jhash(&cmp_arg->id, sizeof(cmp_arg->id), seed);
3488 	default:
3489 		WARN_ON(1);
3490 		return 0;
3491 	}
3492 }
3493 
3494 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
3495 	.head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
3496 	.hashfn	     = mlxsw_sp_nexthop_group_hash,
3497 	.obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
3498 	.obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
3499 };
3500 
3501 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
3502 					 struct mlxsw_sp_nexthop_group *nh_grp)
3503 {
3504 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3505 	    !nh_grp->nhgi->gateway)
3506 		return 0;
3507 
3508 	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
3509 				      &nh_grp->ht_node,
3510 				      mlxsw_sp_nexthop_group_ht_params);
3511 }
3512 
3513 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
3514 					  struct mlxsw_sp_nexthop_group *nh_grp)
3515 {
3516 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6 &&
3517 	    !nh_grp->nhgi->gateway)
3518 		return;
3519 
3520 	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
3521 			       &nh_grp->ht_node,
3522 			       mlxsw_sp_nexthop_group_ht_params);
3523 }
3524 
3525 static struct mlxsw_sp_nexthop_group *
3526 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3527 			       struct fib_info *fi)
3528 {
3529 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3530 
3531 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
3532 	cmp_arg.fi = fi;
3533 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3534 				      &cmp_arg,
3535 				      mlxsw_sp_nexthop_group_ht_params);
3536 }
3537 
3538 static struct mlxsw_sp_nexthop_group *
3539 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3540 			       struct mlxsw_sp_fib6_entry *fib6_entry)
3541 {
3542 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3543 
3544 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
3545 	cmp_arg.fib6_entry = fib6_entry;
3546 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3547 				      &cmp_arg,
3548 				      mlxsw_sp_nexthop_group_ht_params);
3549 }
3550 
3551 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3552 	.key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3553 	.head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3554 	.key_len = sizeof(struct mlxsw_sp_nexthop_key),
3555 };
3556 
3557 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3558 				   struct mlxsw_sp_nexthop *nh)
3559 {
3560 	return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3561 				      &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3562 }
3563 
3564 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3565 				    struct mlxsw_sp_nexthop *nh)
3566 {
3567 	rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3568 			       mlxsw_sp_nexthop_ht_params);
3569 }
3570 
3571 static struct mlxsw_sp_nexthop *
3572 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3573 			struct mlxsw_sp_nexthop_key key)
3574 {
3575 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3576 				      mlxsw_sp_nexthop_ht_params);
3577 }
3578 
3579 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3580 					     enum mlxsw_sp_l3proto proto,
3581 					     u16 vr_id,
3582 					     u32 adj_index, u16 ecmp_size,
3583 					     u32 new_adj_index,
3584 					     u16 new_ecmp_size)
3585 {
3586 	char raleu_pl[MLXSW_REG_RALEU_LEN];
3587 
3588 	mlxsw_reg_raleu_pack(raleu_pl,
3589 			     (enum mlxsw_reg_ralxx_protocol) proto, vr_id,
3590 			     adj_index, ecmp_size, new_adj_index,
3591 			     new_ecmp_size);
3592 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3593 }
3594 
3595 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3596 					  struct mlxsw_sp_nexthop_group *nh_grp,
3597 					  u32 old_adj_index, u16 old_ecmp_size)
3598 {
3599 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
3600 	struct mlxsw_sp_nexthop_group_vr_entry *vr_entry;
3601 	int err;
3602 
3603 	list_for_each_entry(vr_entry, &nh_grp->vr_list, list) {
3604 		err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp,
3605 							vr_entry->key.proto,
3606 							vr_entry->key.vr_id,
3607 							old_adj_index,
3608 							old_ecmp_size,
3609 							nhgi->adj_index,
3610 							nhgi->ecmp_size);
3611 		if (err)
3612 			goto err_mass_update_vr;
3613 	}
3614 	return 0;
3615 
3616 err_mass_update_vr:
3617 	list_for_each_entry_continue_reverse(vr_entry, &nh_grp->vr_list, list)
3618 		mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr_entry->key.proto,
3619 						  vr_entry->key.vr_id,
3620 						  nhgi->adj_index,
3621 						  nhgi->ecmp_size,
3622 						  old_adj_index, old_ecmp_size);
3623 	return err;
3624 }
3625 
3626 static int __mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp,
3627 					 u32 adj_index,
3628 					 struct mlxsw_sp_nexthop *nh,
3629 					 bool force, char *ratr_pl)
3630 {
3631 	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3632 	struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
3633 	enum mlxsw_reg_ratr_op op;
3634 	u16 rif_index;
3635 
3636 	rif_index = rif ? rif->rif_index :
3637 			  mlxsw_sp->router->lb_crif->rif->rif_index;
3638 	op = force ? MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY :
3639 		     MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY_ON_ACTIVITY;
3640 	mlxsw_reg_ratr_pack(ratr_pl, op, true, MLXSW_REG_RATR_TYPE_ETHERNET,
3641 			    adj_index, rif_index);
3642 	switch (nh->action) {
3643 	case MLXSW_SP_NEXTHOP_ACTION_FORWARD:
3644 		mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3645 		break;
3646 	case MLXSW_SP_NEXTHOP_ACTION_DISCARD:
3647 		mlxsw_reg_ratr_trap_action_set(ratr_pl,
3648 					       MLXSW_REG_RATR_TRAP_ACTION_DISCARD_ERRORS);
3649 		break;
3650 	case MLXSW_SP_NEXTHOP_ACTION_TRAP:
3651 		mlxsw_reg_ratr_trap_action_set(ratr_pl,
3652 					       MLXSW_REG_RATR_TRAP_ACTION_TRAP);
3653 		mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
3654 		break;
3655 	default:
3656 		WARN_ON_ONCE(1);
3657 		return -EINVAL;
3658 	}
3659 	if (nh->counter_valid)
3660 		mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3661 	else
3662 		mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3663 
3664 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3665 }
3666 
3667 int mlxsw_sp_nexthop_eth_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3668 				struct mlxsw_sp_nexthop *nh, bool force,
3669 				char *ratr_pl)
3670 {
3671 	int i;
3672 
3673 	for (i = 0; i < nh->num_adj_entries; i++) {
3674 		int err;
3675 
3676 		err = __mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index + i,
3677 						    nh, force, ratr_pl);
3678 		if (err)
3679 			return err;
3680 	}
3681 
3682 	return 0;
3683 }
3684 
3685 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3686 					  u32 adj_index,
3687 					  struct mlxsw_sp_nexthop *nh,
3688 					  bool force, char *ratr_pl)
3689 {
3690 	const struct mlxsw_sp_ipip_ops *ipip_ops;
3691 
3692 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3693 	return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry,
3694 					force, ratr_pl);
3695 }
3696 
3697 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3698 					u32 adj_index,
3699 					struct mlxsw_sp_nexthop *nh, bool force,
3700 					char *ratr_pl)
3701 {
3702 	int i;
3703 
3704 	for (i = 0; i < nh->num_adj_entries; i++) {
3705 		int err;
3706 
3707 		err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3708 						     nh, force, ratr_pl);
3709 		if (err)
3710 			return err;
3711 	}
3712 
3713 	return 0;
3714 }
3715 
3716 static int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3717 				   struct mlxsw_sp_nexthop *nh, bool force,
3718 				   char *ratr_pl)
3719 {
3720 	/* When action is discard or trap, the nexthop must be
3721 	 * programmed as an Ethernet nexthop.
3722 	 */
3723 	if (nh->type == MLXSW_SP_NEXTHOP_TYPE_ETH ||
3724 	    nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD ||
3725 	    nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3726 		return mlxsw_sp_nexthop_eth_update(mlxsw_sp, adj_index, nh,
3727 						   force, ratr_pl);
3728 	else
3729 		return mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index, nh,
3730 						    force, ratr_pl);
3731 }
3732 
3733 static int
3734 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3735 			      struct mlxsw_sp_nexthop_group_info *nhgi,
3736 			      bool reallocate)
3737 {
3738 	char ratr_pl[MLXSW_REG_RATR_LEN];
3739 	u32 adj_index = nhgi->adj_index; /* base */
3740 	struct mlxsw_sp_nexthop *nh;
3741 	int i;
3742 
3743 	for (i = 0; i < nhgi->count; i++) {
3744 		nh = &nhgi->nexthops[i];
3745 
3746 		if (!nh->should_offload) {
3747 			nh->offloaded = 0;
3748 			continue;
3749 		}
3750 
3751 		if (nh->update || reallocate) {
3752 			int err = 0;
3753 
3754 			err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh,
3755 						      true, ratr_pl);
3756 			if (err)
3757 				return err;
3758 			nh->update = 0;
3759 			nh->offloaded = 1;
3760 		}
3761 		adj_index += nh->num_adj_entries;
3762 	}
3763 	return 0;
3764 }
3765 
3766 static int
3767 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3768 				    struct mlxsw_sp_nexthop_group *nh_grp)
3769 {
3770 	struct mlxsw_sp_fib_entry *fib_entry;
3771 	int err;
3772 
3773 	list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3774 		err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3775 		if (err)
3776 			return err;
3777 	}
3778 	return 0;
3779 }
3780 
3781 struct mlxsw_sp_adj_grp_size_range {
3782 	u16 start; /* Inclusive */
3783 	u16 end; /* Inclusive */
3784 };
3785 
3786 /* Ordered by range start value */
3787 static const struct mlxsw_sp_adj_grp_size_range
3788 mlxsw_sp1_adj_grp_size_ranges[] = {
3789 	{ .start = 1, .end = 64 },
3790 	{ .start = 512, .end = 512 },
3791 	{ .start = 1024, .end = 1024 },
3792 	{ .start = 2048, .end = 2048 },
3793 	{ .start = 4096, .end = 4096 },
3794 };
3795 
3796 /* Ordered by range start value */
3797 static const struct mlxsw_sp_adj_grp_size_range
3798 mlxsw_sp2_adj_grp_size_ranges[] = {
3799 	{ .start = 1, .end = 128 },
3800 	{ .start = 256, .end = 256 },
3801 	{ .start = 512, .end = 512 },
3802 	{ .start = 1024, .end = 1024 },
3803 	{ .start = 2048, .end = 2048 },
3804 	{ .start = 4096, .end = 4096 },
3805 };
3806 
3807 static void mlxsw_sp_adj_grp_size_round_up(const struct mlxsw_sp *mlxsw_sp,
3808 					   u16 *p_adj_grp_size)
3809 {
3810 	int i;
3811 
3812 	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
3813 		const struct mlxsw_sp_adj_grp_size_range *size_range;
3814 
3815 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3816 
3817 		if (*p_adj_grp_size >= size_range->start &&
3818 		    *p_adj_grp_size <= size_range->end)
3819 			return;
3820 
3821 		if (*p_adj_grp_size <= size_range->end) {
3822 			*p_adj_grp_size = size_range->end;
3823 			return;
3824 		}
3825 	}
3826 }
3827 
3828 static void mlxsw_sp_adj_grp_size_round_down(const struct mlxsw_sp *mlxsw_sp,
3829 					     u16 *p_adj_grp_size,
3830 					     unsigned int alloc_size)
3831 {
3832 	int i;
3833 
3834 	for (i = mlxsw_sp->router->adj_grp_size_ranges_count - 1; i >= 0; i--) {
3835 		const struct mlxsw_sp_adj_grp_size_range *size_range;
3836 
3837 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
3838 
3839 		if (alloc_size >= size_range->end) {
3840 			*p_adj_grp_size = size_range->end;
3841 			return;
3842 		}
3843 	}
3844 }
3845 
3846 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3847 				     u16 *p_adj_grp_size)
3848 {
3849 	unsigned int alloc_size;
3850 	int err;
3851 
3852 	/* Round up the requested group size to the next size supported
3853 	 * by the device and make sure the request can be satisfied.
3854 	 */
3855 	mlxsw_sp_adj_grp_size_round_up(mlxsw_sp, p_adj_grp_size);
3856 	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3857 					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3858 					      *p_adj_grp_size, &alloc_size);
3859 	if (err)
3860 		return err;
3861 	/* It is possible the allocation results in more allocated
3862 	 * entries than requested. Try to use as much of them as
3863 	 * possible.
3864 	 */
3865 	mlxsw_sp_adj_grp_size_round_down(mlxsw_sp, p_adj_grp_size, alloc_size);
3866 
3867 	return 0;
3868 }
3869 
3870 static void
3871 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group_info *nhgi)
3872 {
3873 	int i, g = 0, sum_norm_weight = 0;
3874 	struct mlxsw_sp_nexthop *nh;
3875 
3876 	for (i = 0; i < nhgi->count; i++) {
3877 		nh = &nhgi->nexthops[i];
3878 
3879 		if (!nh->should_offload)
3880 			continue;
3881 		if (g > 0)
3882 			g = gcd(nh->nh_weight, g);
3883 		else
3884 			g = nh->nh_weight;
3885 	}
3886 
3887 	for (i = 0; i < nhgi->count; i++) {
3888 		nh = &nhgi->nexthops[i];
3889 
3890 		if (!nh->should_offload)
3891 			continue;
3892 		nh->norm_nh_weight = nh->nh_weight / g;
3893 		sum_norm_weight += nh->norm_nh_weight;
3894 	}
3895 
3896 	nhgi->sum_norm_weight = sum_norm_weight;
3897 }
3898 
3899 static void
3900 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group_info *nhgi)
3901 {
3902 	int i, weight = 0, lower_bound = 0;
3903 	int total = nhgi->sum_norm_weight;
3904 	u16 ecmp_size = nhgi->ecmp_size;
3905 
3906 	for (i = 0; i < nhgi->count; i++) {
3907 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
3908 		int upper_bound;
3909 
3910 		if (!nh->should_offload)
3911 			continue;
3912 		weight += nh->norm_nh_weight;
3913 		upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3914 		nh->num_adj_entries = upper_bound - lower_bound;
3915 		lower_bound = upper_bound;
3916 	}
3917 }
3918 
3919 static struct mlxsw_sp_nexthop *
3920 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3921 		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6);
3922 
3923 static void
3924 mlxsw_sp_nexthop4_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3925 					struct mlxsw_sp_nexthop_group *nh_grp)
3926 {
3927 	int i;
3928 
3929 	for (i = 0; i < nh_grp->nhgi->count; i++) {
3930 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
3931 
3932 		if (nh->offloaded)
3933 			nh->key.fib_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3934 		else
3935 			nh->key.fib_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3936 	}
3937 }
3938 
3939 static void
3940 __mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp_nexthop_group *nh_grp,
3941 					  struct mlxsw_sp_fib6_entry *fib6_entry)
3942 {
3943 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3944 
3945 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3946 		struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
3947 		struct mlxsw_sp_nexthop *nh;
3948 
3949 		nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3950 		if (nh && nh->offloaded)
3951 			fib6_nh->fib_nh_flags |= RTNH_F_OFFLOAD;
3952 		else
3953 			fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
3954 	}
3955 }
3956 
3957 static void
3958 mlxsw_sp_nexthop6_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3959 					struct mlxsw_sp_nexthop_group *nh_grp)
3960 {
3961 	struct mlxsw_sp_fib6_entry *fib6_entry;
3962 
3963 	/* Unfortunately, in IPv6 the route and the nexthop are described by
3964 	 * the same struct, so we need to iterate over all the routes using the
3965 	 * nexthop group and set / clear the offload indication for them.
3966 	 */
3967 	list_for_each_entry(fib6_entry, &nh_grp->fib_list,
3968 			    common.nexthop_group_node)
3969 		__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
3970 }
3971 
3972 static void
3973 mlxsw_sp_nexthop_bucket_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3974 					const struct mlxsw_sp_nexthop *nh,
3975 					u16 bucket_index)
3976 {
3977 	struct mlxsw_sp_nexthop_group *nh_grp = nh->nhgi->nh_grp;
3978 	bool offload = false, trap = false;
3979 
3980 	if (nh->offloaded) {
3981 		if (nh->action == MLXSW_SP_NEXTHOP_ACTION_TRAP)
3982 			trap = true;
3983 		else
3984 			offload = true;
3985 	}
3986 	nexthop_bucket_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
3987 				    bucket_index, offload, trap);
3988 }
3989 
3990 static void
3991 mlxsw_sp_nexthop_obj_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
3992 					   struct mlxsw_sp_nexthop_group *nh_grp)
3993 {
3994 	int i;
3995 
3996 	/* Do not update the flags if the nexthop group is being destroyed
3997 	 * since:
3998 	 * 1. The nexthop objects is being deleted, in which case the flags are
3999 	 * irrelevant.
4000 	 * 2. The nexthop group was replaced by a newer group, in which case
4001 	 * the flags of the nexthop object were already updated based on the
4002 	 * new group.
4003 	 */
4004 	if (nh_grp->can_destroy)
4005 		return;
4006 
4007 	nexthop_set_hw_flags(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4008 			     nh_grp->nhgi->adj_index_valid, false);
4009 
4010 	/* Update flags of individual nexthop buckets in case of a resilient
4011 	 * nexthop group.
4012 	 */
4013 	if (!nh_grp->nhgi->is_resilient)
4014 		return;
4015 
4016 	for (i = 0; i < nh_grp->nhgi->count; i++) {
4017 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
4018 
4019 		mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, i);
4020 	}
4021 }
4022 
4023 static void
4024 mlxsw_sp_nexthop_group_offload_refresh(struct mlxsw_sp *mlxsw_sp,
4025 				       struct mlxsw_sp_nexthop_group *nh_grp)
4026 {
4027 	switch (nh_grp->type) {
4028 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4:
4029 		mlxsw_sp_nexthop4_group_offload_refresh(mlxsw_sp, nh_grp);
4030 		break;
4031 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6:
4032 		mlxsw_sp_nexthop6_group_offload_refresh(mlxsw_sp, nh_grp);
4033 		break;
4034 	case MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ:
4035 		mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, nh_grp);
4036 		break;
4037 	}
4038 }
4039 
4040 static int
4041 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
4042 			       struct mlxsw_sp_nexthop_group *nh_grp)
4043 {
4044 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
4045 	u16 ecmp_size, old_ecmp_size;
4046 	struct mlxsw_sp_nexthop *nh;
4047 	bool offload_change = false;
4048 	u32 adj_index;
4049 	bool old_adj_index_valid;
4050 	u32 old_adj_index;
4051 	int i, err2, err;
4052 
4053 	if (!nhgi->gateway)
4054 		return mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4055 
4056 	for (i = 0; i < nhgi->count; i++) {
4057 		nh = &nhgi->nexthops[i];
4058 
4059 		if (nh->should_offload != nh->offloaded) {
4060 			offload_change = true;
4061 			if (nh->should_offload)
4062 				nh->update = 1;
4063 		}
4064 	}
4065 	if (!offload_change) {
4066 		/* Nothing was added or removed, so no need to reallocate. Just
4067 		 * update MAC on existing adjacency indexes.
4068 		 */
4069 		err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, false);
4070 		if (err) {
4071 			dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4072 			goto set_trap;
4073 		}
4074 		/* Flags of individual nexthop buckets might need to be
4075 		 * updated.
4076 		 */
4077 		mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4078 		return 0;
4079 	}
4080 	mlxsw_sp_nexthop_group_normalize(nhgi);
4081 	if (!nhgi->sum_norm_weight) {
4082 		/* No neigh of this group is connected so we just set
4083 		 * the trap and let everthing flow through kernel.
4084 		 */
4085 		err = 0;
4086 		goto set_trap;
4087 	}
4088 
4089 	ecmp_size = nhgi->sum_norm_weight;
4090 	err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
4091 	if (err)
4092 		/* No valid allocation size available. */
4093 		goto set_trap;
4094 
4095 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4096 				  ecmp_size, &adj_index);
4097 	if (err) {
4098 		/* We ran out of KVD linear space, just set the
4099 		 * trap and let everything flow through kernel.
4100 		 */
4101 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
4102 		goto set_trap;
4103 	}
4104 	old_adj_index_valid = nhgi->adj_index_valid;
4105 	old_adj_index = nhgi->adj_index;
4106 	old_ecmp_size = nhgi->ecmp_size;
4107 	nhgi->adj_index_valid = 1;
4108 	nhgi->adj_index = adj_index;
4109 	nhgi->ecmp_size = ecmp_size;
4110 	mlxsw_sp_nexthop_group_rebalance(nhgi);
4111 	err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nhgi, true);
4112 	if (err) {
4113 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
4114 		goto set_trap;
4115 	}
4116 
4117 	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4118 
4119 	if (!old_adj_index_valid) {
4120 		/* The trap was set for fib entries, so we have to call
4121 		 * fib entry update to unset it and use adjacency index.
4122 		 */
4123 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4124 		if (err) {
4125 			dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
4126 			goto set_trap;
4127 		}
4128 		return 0;
4129 	}
4130 
4131 	err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
4132 					     old_adj_index, old_ecmp_size);
4133 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4134 			   old_ecmp_size, old_adj_index);
4135 	if (err) {
4136 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
4137 		goto set_trap;
4138 	}
4139 
4140 	return 0;
4141 
4142 set_trap:
4143 	old_adj_index_valid = nhgi->adj_index_valid;
4144 	nhgi->adj_index_valid = 0;
4145 	for (i = 0; i < nhgi->count; i++) {
4146 		nh = &nhgi->nexthops[i];
4147 		nh->offloaded = 0;
4148 	}
4149 	err2 = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
4150 	if (err2)
4151 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
4152 	mlxsw_sp_nexthop_group_offload_refresh(mlxsw_sp, nh_grp);
4153 	if (old_adj_index_valid)
4154 		mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4155 				   nhgi->ecmp_size, nhgi->adj_index);
4156 	return err;
4157 }
4158 
4159 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
4160 					    bool removing)
4161 {
4162 	if (!removing) {
4163 		nh->action = MLXSW_SP_NEXTHOP_ACTION_FORWARD;
4164 		nh->should_offload = 1;
4165 	} else if (nh->nhgi->is_resilient) {
4166 		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
4167 		nh->should_offload = 1;
4168 	} else {
4169 		nh->should_offload = 0;
4170 	}
4171 	nh->update = 1;
4172 }
4173 
4174 static int
4175 mlxsw_sp_nexthop_dead_neigh_replace(struct mlxsw_sp *mlxsw_sp,
4176 				    struct mlxsw_sp_neigh_entry *neigh_entry)
4177 {
4178 	struct neighbour *n, *old_n = neigh_entry->key.n;
4179 	struct mlxsw_sp_nexthop *nh;
4180 	struct net_device *dev;
4181 	bool entry_connected;
4182 	u8 nud_state, dead;
4183 	int err;
4184 
4185 	nh = list_first_entry(&neigh_entry->nexthop_list,
4186 			      struct mlxsw_sp_nexthop, neigh_list_node);
4187 	dev = mlxsw_sp_nexthop_dev(nh);
4188 
4189 	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev);
4190 	if (!n) {
4191 		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev);
4192 		if (IS_ERR(n))
4193 			return PTR_ERR(n);
4194 		neigh_event_send(n, NULL);
4195 	}
4196 
4197 	mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
4198 	neigh_entry->key.n = n;
4199 	err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4200 	if (err)
4201 		goto err_neigh_entry_insert;
4202 
4203 	read_lock_bh(&n->lock);
4204 	nud_state = n->nud_state;
4205 	dead = n->dead;
4206 	read_unlock_bh(&n->lock);
4207 	entry_connected = nud_state & NUD_VALID && !dead;
4208 
4209 	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4210 			    neigh_list_node) {
4211 		neigh_release(old_n);
4212 		neigh_clone(n);
4213 		__mlxsw_sp_nexthop_neigh_update(nh, !entry_connected);
4214 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4215 	}
4216 
4217 	neigh_release(n);
4218 
4219 	return 0;
4220 
4221 err_neigh_entry_insert:
4222 	neigh_entry->key.n = old_n;
4223 	mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
4224 	neigh_release(n);
4225 	return err;
4226 }
4227 
4228 static void
4229 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
4230 			      struct mlxsw_sp_neigh_entry *neigh_entry,
4231 			      bool removing, bool dead)
4232 {
4233 	struct mlxsw_sp_nexthop *nh;
4234 
4235 	if (list_empty(&neigh_entry->nexthop_list))
4236 		return;
4237 
4238 	if (dead) {
4239 		int err;
4240 
4241 		err = mlxsw_sp_nexthop_dead_neigh_replace(mlxsw_sp,
4242 							  neigh_entry);
4243 		if (err)
4244 			dev_err(mlxsw_sp->bus_info->dev, "Failed to replace dead neigh\n");
4245 		return;
4246 	}
4247 
4248 	list_for_each_entry(nh, &neigh_entry->nexthop_list,
4249 			    neigh_list_node) {
4250 		__mlxsw_sp_nexthop_neigh_update(nh, removing);
4251 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4252 	}
4253 }
4254 
4255 static void mlxsw_sp_nexthop_crif_init(struct mlxsw_sp_nexthop *nh,
4256 				       struct mlxsw_sp_crif *crif)
4257 {
4258 	if (nh->crif)
4259 		return;
4260 
4261 	nh->crif = crif;
4262 	list_add(&nh->crif_list_node, &crif->nexthop_list);
4263 }
4264 
4265 static void mlxsw_sp_nexthop_crif_fini(struct mlxsw_sp_nexthop *nh)
4266 {
4267 	if (!nh->crif)
4268 		return;
4269 
4270 	list_del(&nh->crif_list_node);
4271 	nh->crif = NULL;
4272 }
4273 
4274 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
4275 				       struct mlxsw_sp_nexthop *nh)
4276 {
4277 	struct mlxsw_sp_neigh_entry *neigh_entry;
4278 	struct net_device *dev;
4279 	struct neighbour *n;
4280 	u8 nud_state, dead;
4281 	int err;
4282 
4283 	if (WARN_ON(!nh->crif->rif))
4284 		return 0;
4285 
4286 	if (!nh->nhgi->gateway || nh->neigh_entry)
4287 		return 0;
4288 	dev = mlxsw_sp_nexthop_dev(nh);
4289 
4290 	/* Take a reference of neigh here ensuring that neigh would
4291 	 * not be destructed before the nexthop entry is finished.
4292 	 * The reference is taken either in neigh_lookup() or
4293 	 * in neigh_create() in case n is not found.
4294 	 */
4295 	n = neigh_lookup(nh->neigh_tbl, &nh->gw_addr, dev);
4296 	if (!n) {
4297 		n = neigh_create(nh->neigh_tbl, &nh->gw_addr, dev);
4298 		if (IS_ERR(n))
4299 			return PTR_ERR(n);
4300 		neigh_event_send(n, NULL);
4301 	}
4302 	neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
4303 	if (!neigh_entry) {
4304 		neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
4305 		if (IS_ERR(neigh_entry)) {
4306 			err = -EINVAL;
4307 			goto err_neigh_entry_create;
4308 		}
4309 	}
4310 
4311 	/* If that is the first nexthop connected to that neigh, add to
4312 	 * nexthop_neighs_list
4313 	 */
4314 	if (list_empty(&neigh_entry->nexthop_list))
4315 		list_add_tail(&neigh_entry->nexthop_neighs_list_node,
4316 			      &mlxsw_sp->router->nexthop_neighs_list);
4317 
4318 	nh->neigh_entry = neigh_entry;
4319 	list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
4320 	read_lock_bh(&n->lock);
4321 	nud_state = n->nud_state;
4322 	dead = n->dead;
4323 	read_unlock_bh(&n->lock);
4324 	__mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
4325 
4326 	return 0;
4327 
4328 err_neigh_entry_create:
4329 	neigh_release(n);
4330 	return err;
4331 }
4332 
4333 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
4334 					struct mlxsw_sp_nexthop *nh)
4335 {
4336 	struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
4337 	struct neighbour *n;
4338 
4339 	if (!neigh_entry)
4340 		return;
4341 	n = neigh_entry->key.n;
4342 
4343 	__mlxsw_sp_nexthop_neigh_update(nh, true);
4344 	list_del(&nh->neigh_list_node);
4345 	nh->neigh_entry = NULL;
4346 
4347 	/* If that is the last nexthop connected to that neigh, remove from
4348 	 * nexthop_neighs_list
4349 	 */
4350 	if (list_empty(&neigh_entry->nexthop_list))
4351 		list_del(&neigh_entry->nexthop_neighs_list_node);
4352 
4353 	if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
4354 		mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
4355 
4356 	neigh_release(n);
4357 }
4358 
4359 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
4360 {
4361 	struct net_device *ul_dev;
4362 	bool is_up;
4363 
4364 	rcu_read_lock();
4365 	ul_dev = mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
4366 	is_up = ul_dev ? (ul_dev->flags & IFF_UP) : true;
4367 	rcu_read_unlock();
4368 
4369 	return is_up;
4370 }
4371 
4372 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
4373 				       struct mlxsw_sp_nexthop *nh,
4374 				       struct mlxsw_sp_ipip_entry *ipip_entry)
4375 {
4376 	struct mlxsw_sp_crif *crif;
4377 	bool removing;
4378 
4379 	if (!nh->nhgi->gateway || nh->ipip_entry)
4380 		return;
4381 
4382 	crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, ipip_entry->ol_dev);
4383 	if (WARN_ON(!crif))
4384 		return;
4385 
4386 	nh->ipip_entry = ipip_entry;
4387 	removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
4388 	__mlxsw_sp_nexthop_neigh_update(nh, removing);
4389 	mlxsw_sp_nexthop_crif_init(nh, crif);
4390 }
4391 
4392 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
4393 				       struct mlxsw_sp_nexthop *nh)
4394 {
4395 	struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
4396 
4397 	if (!ipip_entry)
4398 		return;
4399 
4400 	__mlxsw_sp_nexthop_neigh_update(nh, true);
4401 	nh->ipip_entry = NULL;
4402 }
4403 
4404 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4405 					const struct fib_nh *fib_nh,
4406 					enum mlxsw_sp_ipip_type *p_ipipt)
4407 {
4408 	struct net_device *dev = fib_nh->fib_nh_dev;
4409 
4410 	return dev &&
4411 	       fib_nh->nh_parent->fib_type == RTN_UNICAST &&
4412 	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
4413 }
4414 
4415 static int mlxsw_sp_nexthop_type_init(struct mlxsw_sp *mlxsw_sp,
4416 				      struct mlxsw_sp_nexthop *nh,
4417 				      const struct net_device *dev)
4418 {
4419 	const struct mlxsw_sp_ipip_ops *ipip_ops;
4420 	struct mlxsw_sp_ipip_entry *ipip_entry;
4421 	struct mlxsw_sp_crif *crif;
4422 	int err;
4423 
4424 	ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4425 	if (ipip_entry) {
4426 		ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4427 		if (ipip_ops->can_offload(mlxsw_sp, dev)) {
4428 			nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4429 			mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4430 			return 0;
4431 		}
4432 	}
4433 
4434 	nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4435 	crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, dev);
4436 	if (!crif)
4437 		return 0;
4438 
4439 	mlxsw_sp_nexthop_crif_init(nh, crif);
4440 
4441 	if (!crif->rif)
4442 		return 0;
4443 
4444 	err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4445 	if (err)
4446 		goto err_neigh_init;
4447 
4448 	return 0;
4449 
4450 err_neigh_init:
4451 	mlxsw_sp_nexthop_crif_fini(nh);
4452 	return err;
4453 }
4454 
4455 static int mlxsw_sp_nexthop_type_rif_made(struct mlxsw_sp *mlxsw_sp,
4456 					  struct mlxsw_sp_nexthop *nh)
4457 {
4458 	switch (nh->type) {
4459 	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4460 		return mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4461 	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4462 		break;
4463 	}
4464 
4465 	return 0;
4466 }
4467 
4468 static void mlxsw_sp_nexthop_type_rif_gone(struct mlxsw_sp *mlxsw_sp,
4469 					   struct mlxsw_sp_nexthop *nh)
4470 {
4471 	switch (nh->type) {
4472 	case MLXSW_SP_NEXTHOP_TYPE_ETH:
4473 		mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
4474 		break;
4475 	case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4476 		mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
4477 		break;
4478 	}
4479 }
4480 
4481 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
4482 				       struct mlxsw_sp_nexthop *nh)
4483 {
4484 	mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4485 	mlxsw_sp_nexthop_crif_fini(nh);
4486 }
4487 
4488 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
4489 				  struct mlxsw_sp_nexthop_group *nh_grp,
4490 				  struct mlxsw_sp_nexthop *nh,
4491 				  struct fib_nh *fib_nh)
4492 {
4493 	struct net_device *dev = fib_nh->fib_nh_dev;
4494 	struct in_device *in_dev;
4495 	int err;
4496 
4497 	nh->nhgi = nh_grp->nhgi;
4498 	nh->key.fib_nh = fib_nh;
4499 #ifdef CONFIG_IP_ROUTE_MULTIPATH
4500 	nh->nh_weight = fib_nh->fib_nh_weight;
4501 #else
4502 	nh->nh_weight = 1;
4503 #endif
4504 	memcpy(&nh->gw_addr, &fib_nh->fib_nh_gw4, sizeof(fib_nh->fib_nh_gw4));
4505 	nh->neigh_tbl = &arp_tbl;
4506 	err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
4507 	if (err)
4508 		return err;
4509 
4510 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4511 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4512 
4513 	if (!dev)
4514 		return 0;
4515 	nh->ifindex = dev->ifindex;
4516 
4517 	rcu_read_lock();
4518 	in_dev = __in_dev_get_rcu(dev);
4519 	if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
4520 	    fib_nh->fib_nh_flags & RTNH_F_LINKDOWN) {
4521 		rcu_read_unlock();
4522 		return 0;
4523 	}
4524 	rcu_read_unlock();
4525 
4526 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
4527 	if (err)
4528 		goto err_nexthop_neigh_init;
4529 
4530 	return 0;
4531 
4532 err_nexthop_neigh_init:
4533 	list_del(&nh->router_list_node);
4534 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4535 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4536 	return err;
4537 }
4538 
4539 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
4540 				   struct mlxsw_sp_nexthop *nh)
4541 {
4542 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4543 	list_del(&nh->router_list_node);
4544 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4545 	mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
4546 }
4547 
4548 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
4549 				    unsigned long event, struct fib_nh *fib_nh)
4550 {
4551 	struct mlxsw_sp_nexthop_key key;
4552 	struct mlxsw_sp_nexthop *nh;
4553 
4554 	key.fib_nh = fib_nh;
4555 	nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
4556 	if (!nh)
4557 		return;
4558 
4559 	switch (event) {
4560 	case FIB_EVENT_NH_ADD:
4561 		mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, fib_nh->fib_nh_dev);
4562 		break;
4563 	case FIB_EVENT_NH_DEL:
4564 		mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4565 		break;
4566 	}
4567 
4568 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4569 }
4570 
4571 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
4572 					struct mlxsw_sp_rif *rif)
4573 {
4574 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
4575 	struct mlxsw_sp_nexthop *nh;
4576 	bool removing;
4577 
4578 	list_for_each_entry(nh, &rif->crif->nexthop_list, crif_list_node) {
4579 		switch (nh->type) {
4580 		case MLXSW_SP_NEXTHOP_TYPE_ETH:
4581 			removing = false;
4582 			break;
4583 		case MLXSW_SP_NEXTHOP_TYPE_IPIP:
4584 			removing = !mlxsw_sp_ipip_netdev_ul_up(dev);
4585 			break;
4586 		default:
4587 			WARN_ON(1);
4588 			continue;
4589 		}
4590 
4591 		__mlxsw_sp_nexthop_neigh_update(nh, removing);
4592 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4593 	}
4594 }
4595 
4596 static int mlxsw_sp_nexthop_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
4597 					  struct mlxsw_sp_rif *rif)
4598 {
4599 	struct mlxsw_sp_nexthop *nh, *tmp;
4600 	unsigned int n = 0;
4601 	int err;
4602 
4603 	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4604 				 crif_list_node) {
4605 		err = mlxsw_sp_nexthop_type_rif_made(mlxsw_sp, nh);
4606 		if (err)
4607 			goto err_nexthop_type_rif;
4608 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4609 		n++;
4610 	}
4611 
4612 	return 0;
4613 
4614 err_nexthop_type_rif:
4615 	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4616 				 crif_list_node) {
4617 		if (!n--)
4618 			break;
4619 		mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4620 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4621 	}
4622 	return err;
4623 }
4624 
4625 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4626 					   struct mlxsw_sp_rif *rif)
4627 {
4628 	struct mlxsw_sp_nexthop *nh, *tmp;
4629 
4630 	list_for_each_entry_safe(nh, tmp, &rif->crif->nexthop_list,
4631 				 crif_list_node) {
4632 		mlxsw_sp_nexthop_type_rif_gone(mlxsw_sp, nh);
4633 		mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nhgi->nh_grp);
4634 	}
4635 }
4636 
4637 static int mlxsw_sp_adj_trap_entry_init(struct mlxsw_sp *mlxsw_sp)
4638 {
4639 	enum mlxsw_reg_ratr_trap_action trap_action;
4640 	char ratr_pl[MLXSW_REG_RATR_LEN];
4641 	int err;
4642 
4643 	err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4644 				  &mlxsw_sp->router->adj_trap_index);
4645 	if (err)
4646 		return err;
4647 
4648 	trap_action = MLXSW_REG_RATR_TRAP_ACTION_TRAP;
4649 	mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY, true,
4650 			    MLXSW_REG_RATR_TYPE_ETHERNET,
4651 			    mlxsw_sp->router->adj_trap_index,
4652 			    mlxsw_sp->router->lb_crif->rif->rif_index);
4653 	mlxsw_reg_ratr_trap_action_set(ratr_pl, trap_action);
4654 	mlxsw_reg_ratr_trap_id_set(ratr_pl, MLXSW_TRAP_ID_RTR_EGRESS0);
4655 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
4656 	if (err)
4657 		goto err_ratr_write;
4658 
4659 	return 0;
4660 
4661 err_ratr_write:
4662 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4663 			   mlxsw_sp->router->adj_trap_index);
4664 	return err;
4665 }
4666 
4667 static void mlxsw_sp_adj_trap_entry_fini(struct mlxsw_sp *mlxsw_sp)
4668 {
4669 	mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1,
4670 			   mlxsw_sp->router->adj_trap_index);
4671 }
4672 
4673 static int mlxsw_sp_nexthop_group_inc(struct mlxsw_sp *mlxsw_sp)
4674 {
4675 	int err;
4676 
4677 	if (refcount_inc_not_zero(&mlxsw_sp->router->num_groups))
4678 		return 0;
4679 
4680 	err = mlxsw_sp_adj_trap_entry_init(mlxsw_sp);
4681 	if (err)
4682 		return err;
4683 
4684 	refcount_set(&mlxsw_sp->router->num_groups, 1);
4685 
4686 	return 0;
4687 }
4688 
4689 static void mlxsw_sp_nexthop_group_dec(struct mlxsw_sp *mlxsw_sp)
4690 {
4691 	if (!refcount_dec_and_test(&mlxsw_sp->router->num_groups))
4692 		return;
4693 
4694 	mlxsw_sp_adj_trap_entry_fini(mlxsw_sp);
4695 }
4696 
4697 static void
4698 mlxsw_sp_nh_grp_activity_get(struct mlxsw_sp *mlxsw_sp,
4699 			     const struct mlxsw_sp_nexthop_group *nh_grp,
4700 			     unsigned long *activity)
4701 {
4702 	char *ratrad_pl;
4703 	int i, err;
4704 
4705 	ratrad_pl = kmalloc(MLXSW_REG_RATRAD_LEN, GFP_KERNEL);
4706 	if (!ratrad_pl)
4707 		return;
4708 
4709 	mlxsw_reg_ratrad_pack(ratrad_pl, nh_grp->nhgi->adj_index,
4710 			      nh_grp->nhgi->count);
4711 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratrad), ratrad_pl);
4712 	if (err)
4713 		goto out;
4714 
4715 	for (i = 0; i < nh_grp->nhgi->count; i++) {
4716 		if (!mlxsw_reg_ratrad_activity_vector_get(ratrad_pl, i))
4717 			continue;
4718 		bitmap_set(activity, i, 1);
4719 	}
4720 
4721 out:
4722 	kfree(ratrad_pl);
4723 }
4724 
4725 #define MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL 1000 /* ms */
4726 
4727 static void
4728 mlxsw_sp_nh_grp_activity_update(struct mlxsw_sp *mlxsw_sp,
4729 				const struct mlxsw_sp_nexthop_group *nh_grp)
4730 {
4731 	unsigned long *activity;
4732 
4733 	activity = bitmap_zalloc(nh_grp->nhgi->count, GFP_KERNEL);
4734 	if (!activity)
4735 		return;
4736 
4737 	mlxsw_sp_nh_grp_activity_get(mlxsw_sp, nh_grp, activity);
4738 	nexthop_res_grp_activity_update(mlxsw_sp_net(mlxsw_sp), nh_grp->obj.id,
4739 					nh_grp->nhgi->count, activity);
4740 
4741 	bitmap_free(activity);
4742 }
4743 
4744 static void
4745 mlxsw_sp_nh_grp_activity_work_schedule(struct mlxsw_sp *mlxsw_sp)
4746 {
4747 	unsigned int interval = MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL;
4748 
4749 	mlxsw_core_schedule_dw(&mlxsw_sp->router->nh_grp_activity_dw,
4750 			       msecs_to_jiffies(interval));
4751 }
4752 
4753 static void mlxsw_sp_nh_grp_activity_work(struct work_struct *work)
4754 {
4755 	struct mlxsw_sp_nexthop_group_info *nhgi;
4756 	struct mlxsw_sp_router *router;
4757 	bool reschedule = false;
4758 
4759 	router = container_of(work, struct mlxsw_sp_router,
4760 			      nh_grp_activity_dw.work);
4761 
4762 	mutex_lock(&router->lock);
4763 
4764 	list_for_each_entry(nhgi, &router->nh_res_grp_list, list) {
4765 		mlxsw_sp_nh_grp_activity_update(router->mlxsw_sp, nhgi->nh_grp);
4766 		reschedule = true;
4767 	}
4768 
4769 	mutex_unlock(&router->lock);
4770 
4771 	if (!reschedule)
4772 		return;
4773 	mlxsw_sp_nh_grp_activity_work_schedule(router->mlxsw_sp);
4774 }
4775 
4776 static int
4777 mlxsw_sp_nexthop_obj_single_validate(struct mlxsw_sp *mlxsw_sp,
4778 				     const struct nh_notifier_single_info *nh,
4779 				     struct netlink_ext_ack *extack)
4780 {
4781 	int err = -EINVAL;
4782 
4783 	if (nh->is_fdb)
4784 		NL_SET_ERR_MSG_MOD(extack, "FDB nexthops are not supported");
4785 	else if (nh->has_encap)
4786 		NL_SET_ERR_MSG_MOD(extack, "Encapsulating nexthops are not supported");
4787 	else
4788 		err = 0;
4789 
4790 	return err;
4791 }
4792 
4793 static int
4794 mlxsw_sp_nexthop_obj_group_entry_validate(struct mlxsw_sp *mlxsw_sp,
4795 					  const struct nh_notifier_single_info *nh,
4796 					  struct netlink_ext_ack *extack)
4797 {
4798 	int err;
4799 
4800 	err = mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, nh, extack);
4801 	if (err)
4802 		return err;
4803 
4804 	/* Device only nexthops with an IPIP device are programmed as
4805 	 * encapsulating adjacency entries.
4806 	 */
4807 	if (!nh->gw_family && !nh->is_reject &&
4808 	    !mlxsw_sp_netdev_ipip_type(mlxsw_sp, nh->dev, NULL)) {
4809 		NL_SET_ERR_MSG_MOD(extack, "Nexthop group entry does not have a gateway");
4810 		return -EINVAL;
4811 	}
4812 
4813 	return 0;
4814 }
4815 
4816 static int
4817 mlxsw_sp_nexthop_obj_group_validate(struct mlxsw_sp *mlxsw_sp,
4818 				    const struct nh_notifier_grp_info *nh_grp,
4819 				    struct netlink_ext_ack *extack)
4820 {
4821 	int i;
4822 
4823 	if (nh_grp->is_fdb) {
4824 		NL_SET_ERR_MSG_MOD(extack, "FDB nexthop groups are not supported");
4825 		return -EINVAL;
4826 	}
4827 
4828 	for (i = 0; i < nh_grp->num_nh; i++) {
4829 		const struct nh_notifier_single_info *nh;
4830 		int err;
4831 
4832 		nh = &nh_grp->nh_entries[i].nh;
4833 		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4834 								extack);
4835 		if (err)
4836 			return err;
4837 	}
4838 
4839 	return 0;
4840 }
4841 
4842 static int
4843 mlxsw_sp_nexthop_obj_res_group_size_validate(struct mlxsw_sp *mlxsw_sp,
4844 					     const struct nh_notifier_res_table_info *nh_res_table,
4845 					     struct netlink_ext_ack *extack)
4846 {
4847 	unsigned int alloc_size;
4848 	bool valid_size = false;
4849 	int err, i;
4850 
4851 	if (nh_res_table->num_nh_buckets < 32) {
4852 		NL_SET_ERR_MSG_MOD(extack, "Minimum number of buckets is 32");
4853 		return -EINVAL;
4854 	}
4855 
4856 	for (i = 0; i < mlxsw_sp->router->adj_grp_size_ranges_count; i++) {
4857 		const struct mlxsw_sp_adj_grp_size_range *size_range;
4858 
4859 		size_range = &mlxsw_sp->router->adj_grp_size_ranges[i];
4860 
4861 		if (nh_res_table->num_nh_buckets >= size_range->start &&
4862 		    nh_res_table->num_nh_buckets <= size_range->end) {
4863 			valid_size = true;
4864 			break;
4865 		}
4866 	}
4867 
4868 	if (!valid_size) {
4869 		NL_SET_ERR_MSG_MOD(extack, "Invalid number of buckets");
4870 		return -EINVAL;
4871 	}
4872 
4873 	err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
4874 					      MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
4875 					      nh_res_table->num_nh_buckets,
4876 					      &alloc_size);
4877 	if (err || nh_res_table->num_nh_buckets != alloc_size) {
4878 		NL_SET_ERR_MSG_MOD(extack, "Number of buckets does not fit allocation size of any KVDL partition");
4879 		return -EINVAL;
4880 	}
4881 
4882 	return 0;
4883 }
4884 
4885 static int
4886 mlxsw_sp_nexthop_obj_res_group_validate(struct mlxsw_sp *mlxsw_sp,
4887 					const struct nh_notifier_res_table_info *nh_res_table,
4888 					struct netlink_ext_ack *extack)
4889 {
4890 	int err;
4891 	u16 i;
4892 
4893 	err = mlxsw_sp_nexthop_obj_res_group_size_validate(mlxsw_sp,
4894 							   nh_res_table,
4895 							   extack);
4896 	if (err)
4897 		return err;
4898 
4899 	for (i = 0; i < nh_res_table->num_nh_buckets; i++) {
4900 		const struct nh_notifier_single_info *nh;
4901 		int err;
4902 
4903 		nh = &nh_res_table->nhs[i];
4904 		err = mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4905 								extack);
4906 		if (err)
4907 			return err;
4908 	}
4909 
4910 	return 0;
4911 }
4912 
4913 static int mlxsw_sp_nexthop_obj_validate(struct mlxsw_sp *mlxsw_sp,
4914 					 unsigned long event,
4915 					 struct nh_notifier_info *info)
4916 {
4917 	struct nh_notifier_single_info *nh;
4918 
4919 	if (event != NEXTHOP_EVENT_REPLACE &&
4920 	    event != NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE &&
4921 	    event != NEXTHOP_EVENT_BUCKET_REPLACE)
4922 		return 0;
4923 
4924 	switch (info->type) {
4925 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4926 		return mlxsw_sp_nexthop_obj_single_validate(mlxsw_sp, info->nh,
4927 							    info->extack);
4928 	case NH_NOTIFIER_INFO_TYPE_GRP:
4929 		return mlxsw_sp_nexthop_obj_group_validate(mlxsw_sp,
4930 							   info->nh_grp,
4931 							   info->extack);
4932 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4933 		return mlxsw_sp_nexthop_obj_res_group_validate(mlxsw_sp,
4934 							       info->nh_res_table,
4935 							       info->extack);
4936 	case NH_NOTIFIER_INFO_TYPE_RES_BUCKET:
4937 		nh = &info->nh_res_bucket->new_nh;
4938 		return mlxsw_sp_nexthop_obj_group_entry_validate(mlxsw_sp, nh,
4939 								 info->extack);
4940 	default:
4941 		NL_SET_ERR_MSG_MOD(info->extack, "Unsupported nexthop type");
4942 		return -EOPNOTSUPP;
4943 	}
4944 }
4945 
4946 static bool mlxsw_sp_nexthop_obj_is_gateway(struct mlxsw_sp *mlxsw_sp,
4947 					    const struct nh_notifier_info *info)
4948 {
4949 	const struct net_device *dev;
4950 
4951 	switch (info->type) {
4952 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
4953 		dev = info->nh->dev;
4954 		return info->nh->gw_family || info->nh->is_reject ||
4955 		       mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
4956 	case NH_NOTIFIER_INFO_TYPE_GRP:
4957 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
4958 		/* Already validated earlier. */
4959 		return true;
4960 	default:
4961 		return false;
4962 	}
4963 }
4964 
4965 static void mlxsw_sp_nexthop_obj_blackhole_init(struct mlxsw_sp *mlxsw_sp,
4966 						struct mlxsw_sp_nexthop *nh)
4967 {
4968 	nh->action = MLXSW_SP_NEXTHOP_ACTION_DISCARD;
4969 	nh->should_offload = 1;
4970 	/* While nexthops that discard packets do not forward packets
4971 	 * via an egress RIF, they still need to be programmed using a
4972 	 * valid RIF, so use the loopback RIF created during init.
4973 	 */
4974 	nh->crif = mlxsw_sp->router->lb_crif;
4975 }
4976 
4977 static void mlxsw_sp_nexthop_obj_blackhole_fini(struct mlxsw_sp *mlxsw_sp,
4978 						struct mlxsw_sp_nexthop *nh)
4979 {
4980 	nh->crif = NULL;
4981 	nh->should_offload = 0;
4982 }
4983 
4984 static int
4985 mlxsw_sp_nexthop_obj_init(struct mlxsw_sp *mlxsw_sp,
4986 			  struct mlxsw_sp_nexthop_group *nh_grp,
4987 			  struct mlxsw_sp_nexthop *nh,
4988 			  struct nh_notifier_single_info *nh_obj, int weight)
4989 {
4990 	struct net_device *dev = nh_obj->dev;
4991 	int err;
4992 
4993 	nh->nhgi = nh_grp->nhgi;
4994 	nh->nh_weight = weight;
4995 
4996 	switch (nh_obj->gw_family) {
4997 	case AF_INET:
4998 		memcpy(&nh->gw_addr, &nh_obj->ipv4, sizeof(nh_obj->ipv4));
4999 		nh->neigh_tbl = &arp_tbl;
5000 		break;
5001 	case AF_INET6:
5002 		memcpy(&nh->gw_addr, &nh_obj->ipv6, sizeof(nh_obj->ipv6));
5003 #if IS_ENABLED(CONFIG_IPV6)
5004 		nh->neigh_tbl = &nd_tbl;
5005 #endif
5006 		break;
5007 	}
5008 
5009 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5010 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5011 	nh->ifindex = dev->ifindex;
5012 
5013 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
5014 	if (err)
5015 		goto err_type_init;
5016 
5017 	if (nh_obj->is_reject)
5018 		mlxsw_sp_nexthop_obj_blackhole_init(mlxsw_sp, nh);
5019 
5020 	/* In a resilient nexthop group, all the nexthops must be written to
5021 	 * the adjacency table. Even if they do not have a valid neighbour or
5022 	 * RIF.
5023 	 */
5024 	if (nh_grp->nhgi->is_resilient && !nh->should_offload) {
5025 		nh->action = MLXSW_SP_NEXTHOP_ACTION_TRAP;
5026 		nh->should_offload = 1;
5027 	}
5028 
5029 	return 0;
5030 
5031 err_type_init:
5032 	list_del(&nh->router_list_node);
5033 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5034 	return err;
5035 }
5036 
5037 static void mlxsw_sp_nexthop_obj_fini(struct mlxsw_sp *mlxsw_sp,
5038 				      struct mlxsw_sp_nexthop *nh)
5039 {
5040 	if (nh->action == MLXSW_SP_NEXTHOP_ACTION_DISCARD)
5041 		mlxsw_sp_nexthop_obj_blackhole_fini(mlxsw_sp, nh);
5042 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5043 	list_del(&nh->router_list_node);
5044 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5045 	nh->should_offload = 0;
5046 }
5047 
5048 static int
5049 mlxsw_sp_nexthop_obj_group_info_init(struct mlxsw_sp *mlxsw_sp,
5050 				     struct mlxsw_sp_nexthop_group *nh_grp,
5051 				     struct nh_notifier_info *info)
5052 {
5053 	struct mlxsw_sp_nexthop_group_info *nhgi;
5054 	struct mlxsw_sp_nexthop *nh;
5055 	bool is_resilient = false;
5056 	unsigned int nhs;
5057 	int err, i;
5058 
5059 	switch (info->type) {
5060 	case NH_NOTIFIER_INFO_TYPE_SINGLE:
5061 		nhs = 1;
5062 		break;
5063 	case NH_NOTIFIER_INFO_TYPE_GRP:
5064 		nhs = info->nh_grp->num_nh;
5065 		break;
5066 	case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5067 		nhs = info->nh_res_table->num_nh_buckets;
5068 		is_resilient = true;
5069 		break;
5070 	default:
5071 		return -EINVAL;
5072 	}
5073 
5074 	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5075 	if (!nhgi)
5076 		return -ENOMEM;
5077 	nh_grp->nhgi = nhgi;
5078 	nhgi->nh_grp = nh_grp;
5079 	nhgi->gateway = mlxsw_sp_nexthop_obj_is_gateway(mlxsw_sp, info);
5080 	nhgi->is_resilient = is_resilient;
5081 	nhgi->count = nhs;
5082 	for (i = 0; i < nhgi->count; i++) {
5083 		struct nh_notifier_single_info *nh_obj;
5084 		int weight;
5085 
5086 		nh = &nhgi->nexthops[i];
5087 		switch (info->type) {
5088 		case NH_NOTIFIER_INFO_TYPE_SINGLE:
5089 			nh_obj = info->nh;
5090 			weight = 1;
5091 			break;
5092 		case NH_NOTIFIER_INFO_TYPE_GRP:
5093 			nh_obj = &info->nh_grp->nh_entries[i].nh;
5094 			weight = info->nh_grp->nh_entries[i].weight;
5095 			break;
5096 		case NH_NOTIFIER_INFO_TYPE_RES_TABLE:
5097 			nh_obj = &info->nh_res_table->nhs[i];
5098 			weight = 1;
5099 			break;
5100 		default:
5101 			err = -EINVAL;
5102 			goto err_nexthop_obj_init;
5103 		}
5104 		err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj,
5105 						weight);
5106 		if (err)
5107 			goto err_nexthop_obj_init;
5108 	}
5109 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5110 	if (err)
5111 		goto err_group_inc;
5112 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5113 	if (err) {
5114 		NL_SET_ERR_MSG_MOD(info->extack, "Failed to write adjacency entries to the device");
5115 		goto err_group_refresh;
5116 	}
5117 
5118 	/* Add resilient nexthop groups to a list so that the activity of their
5119 	 * nexthop buckets will be periodically queried and cleared.
5120 	 */
5121 	if (nhgi->is_resilient) {
5122 		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
5123 			mlxsw_sp_nh_grp_activity_work_schedule(mlxsw_sp);
5124 		list_add(&nhgi->list, &mlxsw_sp->router->nh_res_grp_list);
5125 	}
5126 
5127 	return 0;
5128 
5129 err_group_refresh:
5130 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5131 err_group_inc:
5132 	i = nhgi->count;
5133 err_nexthop_obj_init:
5134 	for (i--; i >= 0; i--) {
5135 		nh = &nhgi->nexthops[i];
5136 		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5137 	}
5138 	kfree(nhgi);
5139 	return err;
5140 }
5141 
5142 static void
5143 mlxsw_sp_nexthop_obj_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5144 				     struct mlxsw_sp_nexthop_group *nh_grp)
5145 {
5146 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5147 	struct mlxsw_sp_router *router = mlxsw_sp->router;
5148 	int i;
5149 
5150 	if (nhgi->is_resilient) {
5151 		list_del(&nhgi->list);
5152 		if (list_empty(&mlxsw_sp->router->nh_res_grp_list))
5153 			cancel_delayed_work(&router->nh_grp_activity_dw);
5154 	}
5155 
5156 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5157 	for (i = nhgi->count - 1; i >= 0; i--) {
5158 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5159 
5160 		mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5161 	}
5162 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5163 	WARN_ON_ONCE(nhgi->adj_index_valid);
5164 	kfree(nhgi);
5165 }
5166 
5167 static struct mlxsw_sp_nexthop_group *
5168 mlxsw_sp_nexthop_obj_group_create(struct mlxsw_sp *mlxsw_sp,
5169 				  struct nh_notifier_info *info)
5170 {
5171 	struct mlxsw_sp_nexthop_group *nh_grp;
5172 	int err;
5173 
5174 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5175 	if (!nh_grp)
5176 		return ERR_PTR(-ENOMEM);
5177 	INIT_LIST_HEAD(&nh_grp->vr_list);
5178 	err = rhashtable_init(&nh_grp->vr_ht,
5179 			      &mlxsw_sp_nexthop_group_vr_ht_params);
5180 	if (err)
5181 		goto err_nexthop_group_vr_ht_init;
5182 	INIT_LIST_HEAD(&nh_grp->fib_list);
5183 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5184 	nh_grp->obj.id = info->id;
5185 
5186 	err = mlxsw_sp_nexthop_obj_group_info_init(mlxsw_sp, nh_grp, info);
5187 	if (err)
5188 		goto err_nexthop_group_info_init;
5189 
5190 	nh_grp->can_destroy = false;
5191 
5192 	return nh_grp;
5193 
5194 err_nexthop_group_info_init:
5195 	rhashtable_destroy(&nh_grp->vr_ht);
5196 err_nexthop_group_vr_ht_init:
5197 	kfree(nh_grp);
5198 	return ERR_PTR(err);
5199 }
5200 
5201 static void
5202 mlxsw_sp_nexthop_obj_group_destroy(struct mlxsw_sp *mlxsw_sp,
5203 				   struct mlxsw_sp_nexthop_group *nh_grp)
5204 {
5205 	if (!nh_grp->can_destroy)
5206 		return;
5207 	mlxsw_sp_nexthop_obj_group_info_fini(mlxsw_sp, nh_grp);
5208 	WARN_ON_ONCE(!list_empty(&nh_grp->fib_list));
5209 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5210 	rhashtable_destroy(&nh_grp->vr_ht);
5211 	kfree(nh_grp);
5212 }
5213 
5214 static struct mlxsw_sp_nexthop_group *
5215 mlxsw_sp_nexthop_obj_group_lookup(struct mlxsw_sp *mlxsw_sp, u32 id)
5216 {
5217 	struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
5218 
5219 	cmp_arg.type = MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ;
5220 	cmp_arg.id = id;
5221 	return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
5222 				      &cmp_arg,
5223 				      mlxsw_sp_nexthop_group_ht_params);
5224 }
5225 
5226 static int mlxsw_sp_nexthop_obj_group_add(struct mlxsw_sp *mlxsw_sp,
5227 					  struct mlxsw_sp_nexthop_group *nh_grp)
5228 {
5229 	return mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5230 }
5231 
5232 static int
5233 mlxsw_sp_nexthop_obj_group_replace(struct mlxsw_sp *mlxsw_sp,
5234 				   struct mlxsw_sp_nexthop_group *nh_grp,
5235 				   struct mlxsw_sp_nexthop_group *old_nh_grp,
5236 				   struct netlink_ext_ack *extack)
5237 {
5238 	struct mlxsw_sp_nexthop_group_info *old_nhgi = old_nh_grp->nhgi;
5239 	struct mlxsw_sp_nexthop_group_info *new_nhgi = nh_grp->nhgi;
5240 	int err;
5241 
5242 	old_nh_grp->nhgi = new_nhgi;
5243 	new_nhgi->nh_grp = old_nh_grp;
5244 	nh_grp->nhgi = old_nhgi;
5245 	old_nhgi->nh_grp = nh_grp;
5246 
5247 	if (old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5248 		/* Both the old adjacency index and the new one are valid.
5249 		 * Routes are currently using the old one. Tell the device to
5250 		 * replace the old adjacency index with the new one.
5251 		 */
5252 		err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, old_nh_grp,
5253 						     old_nhgi->adj_index,
5254 						     old_nhgi->ecmp_size);
5255 		if (err) {
5256 			NL_SET_ERR_MSG_MOD(extack, "Failed to replace old adjacency index with new one");
5257 			goto err_out;
5258 		}
5259 	} else if (old_nhgi->adj_index_valid && !new_nhgi->adj_index_valid) {
5260 		/* The old adjacency index is valid, while the new one is not.
5261 		 * Iterate over all the routes using the group and change them
5262 		 * to trap packets to the CPU.
5263 		 */
5264 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5265 		if (err) {
5266 			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to trap packets");
5267 			goto err_out;
5268 		}
5269 	} else if (!old_nhgi->adj_index_valid && new_nhgi->adj_index_valid) {
5270 		/* The old adjacency index is invalid, while the new one is.
5271 		 * Iterate over all the routes using the group and change them
5272 		 * to forward packets using the new valid index.
5273 		 */
5274 		err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, old_nh_grp);
5275 		if (err) {
5276 			NL_SET_ERR_MSG_MOD(extack, "Failed to update routes to forward packets");
5277 			goto err_out;
5278 		}
5279 	}
5280 
5281 	/* Make sure the flags are set / cleared based on the new nexthop group
5282 	 * information.
5283 	 */
5284 	mlxsw_sp_nexthop_obj_group_offload_refresh(mlxsw_sp, old_nh_grp);
5285 
5286 	/* At this point 'nh_grp' is just a shell that is not used by anyone
5287 	 * and its nexthop group info is the old info that was just replaced
5288 	 * with the new one. Remove it.
5289 	 */
5290 	nh_grp->can_destroy = true;
5291 	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5292 
5293 	return 0;
5294 
5295 err_out:
5296 	old_nhgi->nh_grp = old_nh_grp;
5297 	nh_grp->nhgi = new_nhgi;
5298 	new_nhgi->nh_grp = nh_grp;
5299 	old_nh_grp->nhgi = old_nhgi;
5300 	return err;
5301 }
5302 
5303 static int mlxsw_sp_nexthop_obj_new(struct mlxsw_sp *mlxsw_sp,
5304 				    struct nh_notifier_info *info)
5305 {
5306 	struct mlxsw_sp_nexthop_group *nh_grp, *old_nh_grp;
5307 	struct netlink_ext_ack *extack = info->extack;
5308 	int err;
5309 
5310 	nh_grp = mlxsw_sp_nexthop_obj_group_create(mlxsw_sp, info);
5311 	if (IS_ERR(nh_grp))
5312 		return PTR_ERR(nh_grp);
5313 
5314 	old_nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5315 	if (!old_nh_grp)
5316 		err = mlxsw_sp_nexthop_obj_group_add(mlxsw_sp, nh_grp);
5317 	else
5318 		err = mlxsw_sp_nexthop_obj_group_replace(mlxsw_sp, nh_grp,
5319 							 old_nh_grp, extack);
5320 
5321 	if (err) {
5322 		nh_grp->can_destroy = true;
5323 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5324 	}
5325 
5326 	return err;
5327 }
5328 
5329 static void mlxsw_sp_nexthop_obj_del(struct mlxsw_sp *mlxsw_sp,
5330 				     struct nh_notifier_info *info)
5331 {
5332 	struct mlxsw_sp_nexthop_group *nh_grp;
5333 
5334 	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5335 	if (!nh_grp)
5336 		return;
5337 
5338 	nh_grp->can_destroy = true;
5339 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5340 
5341 	/* If the group still has routes using it, then defer the delete
5342 	 * operation until the last route using it is deleted.
5343 	 */
5344 	if (!list_empty(&nh_grp->fib_list))
5345 		return;
5346 	mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5347 }
5348 
5349 static int mlxsw_sp_nexthop_obj_bucket_query(struct mlxsw_sp *mlxsw_sp,
5350 					     u32 adj_index, char *ratr_pl)
5351 {
5352 	MLXSW_REG_ZERO(ratr, ratr_pl);
5353 	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5354 	mlxsw_reg_ratr_adjacency_index_low_set(ratr_pl, adj_index);
5355 	mlxsw_reg_ratr_adjacency_index_high_set(ratr_pl, adj_index >> 16);
5356 
5357 	return mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
5358 }
5359 
5360 static int mlxsw_sp_nexthop_obj_bucket_compare(char *ratr_pl, char *ratr_pl_new)
5361 {
5362 	/* Clear the opcode and activity on both the old and new payload as
5363 	 * they are irrelevant for the comparison.
5364 	 */
5365 	mlxsw_reg_ratr_op_set(ratr_pl, MLXSW_REG_RATR_OP_QUERY_READ);
5366 	mlxsw_reg_ratr_a_set(ratr_pl, 0);
5367 	mlxsw_reg_ratr_op_set(ratr_pl_new, MLXSW_REG_RATR_OP_QUERY_READ);
5368 	mlxsw_reg_ratr_a_set(ratr_pl_new, 0);
5369 
5370 	/* If the contents of the adjacency entry are consistent with the
5371 	 * replacement request, then replacement was successful.
5372 	 */
5373 	if (!memcmp(ratr_pl, ratr_pl_new, MLXSW_REG_RATR_LEN))
5374 		return 0;
5375 
5376 	return -EINVAL;
5377 }
5378 
5379 static int
5380 mlxsw_sp_nexthop_obj_bucket_adj_update(struct mlxsw_sp *mlxsw_sp,
5381 				       struct mlxsw_sp_nexthop *nh,
5382 				       struct nh_notifier_info *info)
5383 {
5384 	u16 bucket_index = info->nh_res_bucket->bucket_index;
5385 	struct netlink_ext_ack *extack = info->extack;
5386 	bool force = info->nh_res_bucket->force;
5387 	char ratr_pl_new[MLXSW_REG_RATR_LEN];
5388 	char ratr_pl[MLXSW_REG_RATR_LEN];
5389 	u32 adj_index;
5390 	int err;
5391 
5392 	/* No point in trying an atomic replacement if the idle timer interval
5393 	 * is smaller than the interval in which we query and clear activity.
5394 	 */
5395 	if (!force && info->nh_res_bucket->idle_timer_ms <
5396 	    MLXSW_SP_NH_GRP_ACTIVITY_UPDATE_INTERVAL)
5397 		force = true;
5398 
5399 	adj_index = nh->nhgi->adj_index + bucket_index;
5400 	err = mlxsw_sp_nexthop_update(mlxsw_sp, adj_index, nh, force, ratr_pl);
5401 	if (err) {
5402 		NL_SET_ERR_MSG_MOD(extack, "Failed to overwrite nexthop bucket");
5403 		return err;
5404 	}
5405 
5406 	if (!force) {
5407 		err = mlxsw_sp_nexthop_obj_bucket_query(mlxsw_sp, adj_index,
5408 							ratr_pl_new);
5409 		if (err) {
5410 			NL_SET_ERR_MSG_MOD(extack, "Failed to query nexthop bucket state after replacement. State might be inconsistent");
5411 			return err;
5412 		}
5413 
5414 		err = mlxsw_sp_nexthop_obj_bucket_compare(ratr_pl, ratr_pl_new);
5415 		if (err) {
5416 			NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket was not replaced because it was active during replacement");
5417 			return err;
5418 		}
5419 	}
5420 
5421 	nh->update = 0;
5422 	nh->offloaded = 1;
5423 	mlxsw_sp_nexthop_bucket_offload_refresh(mlxsw_sp, nh, bucket_index);
5424 
5425 	return 0;
5426 }
5427 
5428 static int mlxsw_sp_nexthop_obj_bucket_replace(struct mlxsw_sp *mlxsw_sp,
5429 					       struct nh_notifier_info *info)
5430 {
5431 	u16 bucket_index = info->nh_res_bucket->bucket_index;
5432 	struct netlink_ext_ack *extack = info->extack;
5433 	struct mlxsw_sp_nexthop_group_info *nhgi;
5434 	struct nh_notifier_single_info *nh_obj;
5435 	struct mlxsw_sp_nexthop_group *nh_grp;
5436 	struct mlxsw_sp_nexthop *nh;
5437 	int err;
5438 
5439 	nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, info->id);
5440 	if (!nh_grp) {
5441 		NL_SET_ERR_MSG_MOD(extack, "Nexthop group was not found");
5442 		return -EINVAL;
5443 	}
5444 
5445 	nhgi = nh_grp->nhgi;
5446 
5447 	if (bucket_index >= nhgi->count) {
5448 		NL_SET_ERR_MSG_MOD(extack, "Nexthop bucket index out of range");
5449 		return -EINVAL;
5450 	}
5451 
5452 	nh = &nhgi->nexthops[bucket_index];
5453 	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5454 
5455 	nh_obj = &info->nh_res_bucket->new_nh;
5456 	err = mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5457 	if (err) {
5458 		NL_SET_ERR_MSG_MOD(extack, "Failed to initialize nexthop object for nexthop bucket replacement");
5459 		goto err_nexthop_obj_init;
5460 	}
5461 
5462 	err = mlxsw_sp_nexthop_obj_bucket_adj_update(mlxsw_sp, nh, info);
5463 	if (err)
5464 		goto err_nexthop_obj_bucket_adj_update;
5465 
5466 	return 0;
5467 
5468 err_nexthop_obj_bucket_adj_update:
5469 	mlxsw_sp_nexthop_obj_fini(mlxsw_sp, nh);
5470 err_nexthop_obj_init:
5471 	nh_obj = &info->nh_res_bucket->old_nh;
5472 	mlxsw_sp_nexthop_obj_init(mlxsw_sp, nh_grp, nh, nh_obj, 1);
5473 	/* The old adjacency entry was not overwritten */
5474 	nh->update = 0;
5475 	nh->offloaded = 1;
5476 	return err;
5477 }
5478 
5479 static int mlxsw_sp_nexthop_obj_event(struct notifier_block *nb,
5480 				      unsigned long event, void *ptr)
5481 {
5482 	struct nh_notifier_info *info = ptr;
5483 	struct mlxsw_sp_router *router;
5484 	int err = 0;
5485 
5486 	router = container_of(nb, struct mlxsw_sp_router, nexthop_nb);
5487 	err = mlxsw_sp_nexthop_obj_validate(router->mlxsw_sp, event, info);
5488 	if (err)
5489 		goto out;
5490 
5491 	mutex_lock(&router->lock);
5492 
5493 	switch (event) {
5494 	case NEXTHOP_EVENT_REPLACE:
5495 		err = mlxsw_sp_nexthop_obj_new(router->mlxsw_sp, info);
5496 		break;
5497 	case NEXTHOP_EVENT_DEL:
5498 		mlxsw_sp_nexthop_obj_del(router->mlxsw_sp, info);
5499 		break;
5500 	case NEXTHOP_EVENT_BUCKET_REPLACE:
5501 		err = mlxsw_sp_nexthop_obj_bucket_replace(router->mlxsw_sp,
5502 							  info);
5503 		break;
5504 	default:
5505 		break;
5506 	}
5507 
5508 	mutex_unlock(&router->lock);
5509 
5510 out:
5511 	return notifier_from_errno(err);
5512 }
5513 
5514 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5515 				   struct fib_info *fi)
5516 {
5517 	const struct fib_nh *nh = fib_info_nh(fi, 0);
5518 
5519 	return nh->fib_nh_gw_family ||
5520 	       mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, nh, NULL);
5521 }
5522 
5523 static int
5524 mlxsw_sp_nexthop4_group_info_init(struct mlxsw_sp *mlxsw_sp,
5525 				  struct mlxsw_sp_nexthop_group *nh_grp)
5526 {
5527 	unsigned int nhs = fib_info_num_path(nh_grp->ipv4.fi);
5528 	struct mlxsw_sp_nexthop_group_info *nhgi;
5529 	struct mlxsw_sp_nexthop *nh;
5530 	int err, i;
5531 
5532 	nhgi = kzalloc(struct_size(nhgi, nexthops, nhs), GFP_KERNEL);
5533 	if (!nhgi)
5534 		return -ENOMEM;
5535 	nh_grp->nhgi = nhgi;
5536 	nhgi->nh_grp = nh_grp;
5537 	nhgi->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, nh_grp->ipv4.fi);
5538 	nhgi->count = nhs;
5539 	for (i = 0; i < nhgi->count; i++) {
5540 		struct fib_nh *fib_nh;
5541 
5542 		nh = &nhgi->nexthops[i];
5543 		fib_nh = fib_info_nh(nh_grp->ipv4.fi, i);
5544 		err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
5545 		if (err)
5546 			goto err_nexthop4_init;
5547 	}
5548 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
5549 	if (err)
5550 		goto err_group_inc;
5551 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5552 	if (err)
5553 		goto err_group_refresh;
5554 
5555 	return 0;
5556 
5557 err_group_refresh:
5558 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5559 err_group_inc:
5560 	i = nhgi->count;
5561 err_nexthop4_init:
5562 	for (i--; i >= 0; i--) {
5563 		nh = &nhgi->nexthops[i];
5564 		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5565 	}
5566 	kfree(nhgi);
5567 	return err;
5568 }
5569 
5570 static void
5571 mlxsw_sp_nexthop4_group_info_fini(struct mlxsw_sp *mlxsw_sp,
5572 				  struct mlxsw_sp_nexthop_group *nh_grp)
5573 {
5574 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
5575 	int i;
5576 
5577 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
5578 	for (i = nhgi->count - 1; i >= 0; i--) {
5579 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
5580 
5581 		mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
5582 	}
5583 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5584 	WARN_ON_ONCE(nhgi->adj_index_valid);
5585 	kfree(nhgi);
5586 }
5587 
5588 static struct mlxsw_sp_nexthop_group *
5589 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
5590 {
5591 	struct mlxsw_sp_nexthop_group *nh_grp;
5592 	int err;
5593 
5594 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
5595 	if (!nh_grp)
5596 		return ERR_PTR(-ENOMEM);
5597 	INIT_LIST_HEAD(&nh_grp->vr_list);
5598 	err = rhashtable_init(&nh_grp->vr_ht,
5599 			      &mlxsw_sp_nexthop_group_vr_ht_params);
5600 	if (err)
5601 		goto err_nexthop_group_vr_ht_init;
5602 	INIT_LIST_HEAD(&nh_grp->fib_list);
5603 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV4;
5604 	nh_grp->ipv4.fi = fi;
5605 	fib_info_hold(fi);
5606 
5607 	err = mlxsw_sp_nexthop4_group_info_init(mlxsw_sp, nh_grp);
5608 	if (err)
5609 		goto err_nexthop_group_info_init;
5610 
5611 	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5612 	if (err)
5613 		goto err_nexthop_group_insert;
5614 
5615 	nh_grp->can_destroy = true;
5616 
5617 	return nh_grp;
5618 
5619 err_nexthop_group_insert:
5620 	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5621 err_nexthop_group_info_init:
5622 	fib_info_put(fi);
5623 	rhashtable_destroy(&nh_grp->vr_ht);
5624 err_nexthop_group_vr_ht_init:
5625 	kfree(nh_grp);
5626 	return ERR_PTR(err);
5627 }
5628 
5629 static void
5630 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
5631 				struct mlxsw_sp_nexthop_group *nh_grp)
5632 {
5633 	if (!nh_grp->can_destroy)
5634 		return;
5635 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5636 	mlxsw_sp_nexthop4_group_info_fini(mlxsw_sp, nh_grp);
5637 	fib_info_put(nh_grp->ipv4.fi);
5638 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
5639 	rhashtable_destroy(&nh_grp->vr_ht);
5640 	kfree(nh_grp);
5641 }
5642 
5643 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
5644 				       struct mlxsw_sp_fib_entry *fib_entry,
5645 				       struct fib_info *fi)
5646 {
5647 	struct mlxsw_sp_nexthop_group *nh_grp;
5648 
5649 	if (fi->nh) {
5650 		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
5651 							   fi->nh->id);
5652 		if (WARN_ON_ONCE(!nh_grp))
5653 			return -EINVAL;
5654 		goto out;
5655 	}
5656 
5657 	nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
5658 	if (!nh_grp) {
5659 		nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
5660 		if (IS_ERR(nh_grp))
5661 			return PTR_ERR(nh_grp);
5662 	}
5663 out:
5664 	list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
5665 	fib_entry->nh_group = nh_grp;
5666 	return 0;
5667 }
5668 
5669 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
5670 					struct mlxsw_sp_fib_entry *fib_entry)
5671 {
5672 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5673 
5674 	list_del(&fib_entry->nexthop_group_node);
5675 	if (!list_empty(&nh_grp->fib_list))
5676 		return;
5677 
5678 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
5679 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
5680 		return;
5681 	}
5682 
5683 	mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
5684 }
5685 
5686 static bool
5687 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5688 {
5689 	struct mlxsw_sp_fib4_entry *fib4_entry;
5690 
5691 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5692 				  common);
5693 	return !fib4_entry->dscp;
5694 }
5695 
5696 static bool
5697 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
5698 {
5699 	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5700 
5701 	switch (fib_entry->fib_node->fib->proto) {
5702 	case MLXSW_SP_L3_PROTO_IPV4:
5703 		if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
5704 			return false;
5705 		break;
5706 	case MLXSW_SP_L3_PROTO_IPV6:
5707 		break;
5708 	}
5709 
5710 	switch (fib_entry->type) {
5711 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
5712 		return !!nh_group->nhgi->adj_index_valid;
5713 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
5714 		return !!mlxsw_sp_nhgi_rif(nh_group->nhgi);
5715 	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
5716 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
5717 	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
5718 		return true;
5719 	default:
5720 		return false;
5721 	}
5722 }
5723 
5724 static struct mlxsw_sp_nexthop *
5725 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
5726 		     const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
5727 {
5728 	int i;
5729 
5730 	for (i = 0; i < nh_grp->nhgi->count; i++) {
5731 		struct mlxsw_sp_nexthop *nh = &nh_grp->nhgi->nexthops[i];
5732 		struct net_device *dev = mlxsw_sp_nexthop_dev(nh);
5733 		struct fib6_info *rt = mlxsw_sp_rt6->rt;
5734 
5735 		if (dev && dev == rt->fib6_nh->fib_nh_dev &&
5736 		    ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
5737 				    &rt->fib6_nh->fib_nh_gw6))
5738 			return nh;
5739 	}
5740 
5741 	return NULL;
5742 }
5743 
5744 static void
5745 mlxsw_sp_fib4_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5746 				      struct fib_entry_notifier_info *fen_info)
5747 {
5748 	u32 *p_dst = (u32 *) &fen_info->dst;
5749 	struct fib_rt_info fri;
5750 
5751 	fri.fi = fen_info->fi;
5752 	fri.tb_id = fen_info->tb_id;
5753 	fri.dst = cpu_to_be32(*p_dst);
5754 	fri.dst_len = fen_info->dst_len;
5755 	fri.dscp = fen_info->dscp;
5756 	fri.type = fen_info->type;
5757 	fri.offload = false;
5758 	fri.trap = false;
5759 	fri.offload_failed = true;
5760 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5761 }
5762 
5763 static void
5764 mlxsw_sp_fib4_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5765 				 struct mlxsw_sp_fib_entry *fib_entry)
5766 {
5767 	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5768 	int dst_len = fib_entry->fib_node->key.prefix_len;
5769 	struct mlxsw_sp_fib4_entry *fib4_entry;
5770 	struct fib_rt_info fri;
5771 	bool should_offload;
5772 
5773 	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5774 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5775 				  common);
5776 	fri.fi = fib4_entry->fi;
5777 	fri.tb_id = fib4_entry->tb_id;
5778 	fri.dst = cpu_to_be32(*p_dst);
5779 	fri.dst_len = dst_len;
5780 	fri.dscp = fib4_entry->dscp;
5781 	fri.type = fib4_entry->type;
5782 	fri.offload = should_offload;
5783 	fri.trap = !should_offload;
5784 	fri.offload_failed = false;
5785 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5786 }
5787 
5788 static void
5789 mlxsw_sp_fib4_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5790 				   struct mlxsw_sp_fib_entry *fib_entry)
5791 {
5792 	u32 *p_dst = (u32 *) fib_entry->fib_node->key.addr;
5793 	int dst_len = fib_entry->fib_node->key.prefix_len;
5794 	struct mlxsw_sp_fib4_entry *fib4_entry;
5795 	struct fib_rt_info fri;
5796 
5797 	fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
5798 				  common);
5799 	fri.fi = fib4_entry->fi;
5800 	fri.tb_id = fib4_entry->tb_id;
5801 	fri.dst = cpu_to_be32(*p_dst);
5802 	fri.dst_len = dst_len;
5803 	fri.dscp = fib4_entry->dscp;
5804 	fri.type = fib4_entry->type;
5805 	fri.offload = false;
5806 	fri.trap = false;
5807 	fri.offload_failed = false;
5808 	fib_alias_hw_flags_set(mlxsw_sp_net(mlxsw_sp), &fri);
5809 }
5810 
5811 #if IS_ENABLED(CONFIG_IPV6)
5812 static void
5813 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5814 				      struct fib6_info **rt_arr,
5815 				      unsigned int nrt6)
5816 {
5817 	int i;
5818 
5819 	/* In IPv6 a multipath route is represented using multiple routes, so
5820 	 * we need to set the flags on all of them.
5821 	 */
5822 	for (i = 0; i < nrt6; i++)
5823 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), rt_arr[i],
5824 				       false, false, true);
5825 }
5826 #else
5827 static void
5828 mlxsw_sp_fib6_offload_failed_flag_set(struct mlxsw_sp *mlxsw_sp,
5829 				      struct fib6_info **rt_arr,
5830 				      unsigned int nrt6)
5831 {
5832 }
5833 #endif
5834 
5835 #if IS_ENABLED(CONFIG_IPV6)
5836 static void
5837 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5838 				 struct mlxsw_sp_fib_entry *fib_entry)
5839 {
5840 	struct mlxsw_sp_fib6_entry *fib6_entry;
5841 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5842 	bool should_offload;
5843 
5844 	should_offload = mlxsw_sp_fib_entry_should_offload(fib_entry);
5845 
5846 	/* In IPv6 a multipath route is represented using multiple routes, so
5847 	 * we need to set the flags on all of them.
5848 	 */
5849 	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5850 				  common);
5851 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5852 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5853 				       should_offload, !should_offload, false);
5854 }
5855 #else
5856 static void
5857 mlxsw_sp_fib6_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5858 				 struct mlxsw_sp_fib_entry *fib_entry)
5859 {
5860 }
5861 #endif
5862 
5863 #if IS_ENABLED(CONFIG_IPV6)
5864 static void
5865 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5866 				   struct mlxsw_sp_fib_entry *fib_entry)
5867 {
5868 	struct mlxsw_sp_fib6_entry *fib6_entry;
5869 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5870 
5871 	fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
5872 				  common);
5873 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list)
5874 		fib6_info_hw_flags_set(mlxsw_sp_net(mlxsw_sp), mlxsw_sp_rt6->rt,
5875 				       false, false, false);
5876 }
5877 #else
5878 static void
5879 mlxsw_sp_fib6_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5880 				   struct mlxsw_sp_fib_entry *fib_entry)
5881 {
5882 }
5883 #endif
5884 
5885 static void
5886 mlxsw_sp_fib_entry_hw_flags_set(struct mlxsw_sp *mlxsw_sp,
5887 				struct mlxsw_sp_fib_entry *fib_entry)
5888 {
5889 	switch (fib_entry->fib_node->fib->proto) {
5890 	case MLXSW_SP_L3_PROTO_IPV4:
5891 		mlxsw_sp_fib4_entry_hw_flags_set(mlxsw_sp, fib_entry);
5892 		break;
5893 	case MLXSW_SP_L3_PROTO_IPV6:
5894 		mlxsw_sp_fib6_entry_hw_flags_set(mlxsw_sp, fib_entry);
5895 		break;
5896 	}
5897 }
5898 
5899 static void
5900 mlxsw_sp_fib_entry_hw_flags_clear(struct mlxsw_sp *mlxsw_sp,
5901 				  struct mlxsw_sp_fib_entry *fib_entry)
5902 {
5903 	switch (fib_entry->fib_node->fib->proto) {
5904 	case MLXSW_SP_L3_PROTO_IPV4:
5905 		mlxsw_sp_fib4_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5906 		break;
5907 	case MLXSW_SP_L3_PROTO_IPV6:
5908 		mlxsw_sp_fib6_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5909 		break;
5910 	}
5911 }
5912 
5913 static void
5914 mlxsw_sp_fib_entry_hw_flags_refresh(struct mlxsw_sp *mlxsw_sp,
5915 				    struct mlxsw_sp_fib_entry *fib_entry,
5916 				    enum mlxsw_reg_ralue_op op)
5917 {
5918 	switch (op) {
5919 	case MLXSW_REG_RALUE_OP_WRITE_WRITE:
5920 		mlxsw_sp_fib_entry_hw_flags_set(mlxsw_sp, fib_entry);
5921 		break;
5922 	case MLXSW_REG_RALUE_OP_WRITE_DELETE:
5923 		mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, fib_entry);
5924 		break;
5925 	default:
5926 		break;
5927 	}
5928 }
5929 
5930 static void
5931 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
5932 			      const struct mlxsw_sp_fib_entry *fib_entry,
5933 			      enum mlxsw_reg_ralue_op op)
5934 {
5935 	struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
5936 	enum mlxsw_reg_ralxx_protocol proto;
5937 	u32 *p_dip;
5938 
5939 	proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
5940 
5941 	switch (fib->proto) {
5942 	case MLXSW_SP_L3_PROTO_IPV4:
5943 		p_dip = (u32 *) fib_entry->fib_node->key.addr;
5944 		mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
5945 				      fib_entry->fib_node->key.prefix_len,
5946 				      *p_dip);
5947 		break;
5948 	case MLXSW_SP_L3_PROTO_IPV6:
5949 		mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
5950 				      fib_entry->fib_node->key.prefix_len,
5951 				      fib_entry->fib_node->key.addr);
5952 		break;
5953 	}
5954 }
5955 
5956 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
5957 					struct mlxsw_sp_fib_entry *fib_entry,
5958 					enum mlxsw_reg_ralue_op op)
5959 {
5960 	struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
5961 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_group->nhgi;
5962 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5963 	enum mlxsw_reg_ralue_trap_action trap_action;
5964 	u16 trap_id = 0;
5965 	u32 adjacency_index = 0;
5966 	u16 ecmp_size = 0;
5967 
5968 	/* In case the nexthop group adjacency index is valid, use it
5969 	 * with provided ECMP size. Otherwise, setup trap and pass
5970 	 * traffic to kernel.
5971 	 */
5972 	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
5973 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5974 		adjacency_index = nhgi->adj_index;
5975 		ecmp_size = nhgi->ecmp_size;
5976 	} else if (!nhgi->adj_index_valid && nhgi->count &&
5977 		   mlxsw_sp_nhgi_rif(nhgi)) {
5978 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
5979 		adjacency_index = mlxsw_sp->router->adj_trap_index;
5980 		ecmp_size = 1;
5981 	} else {
5982 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
5983 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
5984 	}
5985 
5986 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
5987 	mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
5988 					adjacency_index, ecmp_size);
5989 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
5990 }
5991 
5992 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
5993 				       struct mlxsw_sp_fib_entry *fib_entry,
5994 				       enum mlxsw_reg_ralue_op op)
5995 {
5996 	struct mlxsw_sp_rif *rif = mlxsw_sp_nhgi_rif(fib_entry->nh_group->nhgi);
5997 	enum mlxsw_reg_ralue_trap_action trap_action;
5998 	char ralue_pl[MLXSW_REG_RALUE_LEN];
5999 	u16 trap_id = 0;
6000 	u16 rif_index = 0;
6001 
6002 	if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
6003 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
6004 		rif_index = rif->rif_index;
6005 	} else {
6006 		trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6007 		trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
6008 	}
6009 
6010 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6011 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
6012 				       rif_index);
6013 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6014 }
6015 
6016 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
6017 				      struct mlxsw_sp_fib_entry *fib_entry,
6018 				      enum mlxsw_reg_ralue_op op)
6019 {
6020 	char ralue_pl[MLXSW_REG_RALUE_LEN];
6021 
6022 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6023 	mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
6024 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6025 }
6026 
6027 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
6028 					   struct mlxsw_sp_fib_entry *fib_entry,
6029 					   enum mlxsw_reg_ralue_op op)
6030 {
6031 	enum mlxsw_reg_ralue_trap_action trap_action;
6032 	char ralue_pl[MLXSW_REG_RALUE_LEN];
6033 
6034 	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
6035 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6036 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0);
6037 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6038 }
6039 
6040 static int
6041 mlxsw_sp_fib_entry_op_unreachable(struct mlxsw_sp *mlxsw_sp,
6042 				  struct mlxsw_sp_fib_entry *fib_entry,
6043 				  enum mlxsw_reg_ralue_op op)
6044 {
6045 	enum mlxsw_reg_ralue_trap_action trap_action;
6046 	char ralue_pl[MLXSW_REG_RALUE_LEN];
6047 	u16 trap_id;
6048 
6049 	trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
6050 	trap_id = MLXSW_TRAP_ID_RTR_INGRESS1;
6051 
6052 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6053 	mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id, 0);
6054 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6055 }
6056 
6057 static int
6058 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
6059 				 struct mlxsw_sp_fib_entry *fib_entry,
6060 				 enum mlxsw_reg_ralue_op op)
6061 {
6062 	struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
6063 	const struct mlxsw_sp_ipip_ops *ipip_ops;
6064 	char ralue_pl[MLXSW_REG_RALUE_LEN];
6065 	int err;
6066 
6067 	if (WARN_ON(!ipip_entry))
6068 		return -EINVAL;
6069 
6070 	ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
6071 	err = ipip_ops->decap_config(mlxsw_sp, ipip_entry,
6072 				     fib_entry->decap.tunnel_index);
6073 	if (err)
6074 		return err;
6075 
6076 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6077 	mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
6078 					   fib_entry->decap.tunnel_index);
6079 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6080 }
6081 
6082 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
6083 					   struct mlxsw_sp_fib_entry *fib_entry,
6084 					   enum mlxsw_reg_ralue_op op)
6085 {
6086 	char ralue_pl[MLXSW_REG_RALUE_LEN];
6087 
6088 	mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
6089 	mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
6090 					   fib_entry->decap.tunnel_index);
6091 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
6092 }
6093 
6094 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6095 				   struct mlxsw_sp_fib_entry *fib_entry,
6096 				   enum mlxsw_reg_ralue_op op)
6097 {
6098 	switch (fib_entry->type) {
6099 	case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
6100 		return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
6101 	case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
6102 		return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
6103 	case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
6104 		return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
6105 	case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
6106 		return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op);
6107 	case MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE:
6108 		return mlxsw_sp_fib_entry_op_unreachable(mlxsw_sp, fib_entry,
6109 							 op);
6110 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6111 		return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
6112 							fib_entry, op);
6113 	case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
6114 		return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op);
6115 	}
6116 	return -EINVAL;
6117 }
6118 
6119 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
6120 				 struct mlxsw_sp_fib_entry *fib_entry,
6121 				 enum mlxsw_reg_ralue_op op)
6122 {
6123 	int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
6124 
6125 	if (err)
6126 		return err;
6127 
6128 	mlxsw_sp_fib_entry_hw_flags_refresh(mlxsw_sp, fib_entry, op);
6129 
6130 	return err;
6131 }
6132 
6133 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
6134 				     struct mlxsw_sp_fib_entry *fib_entry)
6135 {
6136 	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6137 				     MLXSW_REG_RALUE_OP_WRITE_WRITE);
6138 }
6139 
6140 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
6141 				  struct mlxsw_sp_fib_entry *fib_entry)
6142 {
6143 	return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
6144 				     MLXSW_REG_RALUE_OP_WRITE_DELETE);
6145 }
6146 
6147 static int
6148 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
6149 			     const struct fib_entry_notifier_info *fen_info,
6150 			     struct mlxsw_sp_fib_entry *fib_entry)
6151 {
6152 	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
6153 	union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
6154 	struct mlxsw_sp_router *router = mlxsw_sp->router;
6155 	u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
6156 	int ifindex = nhgi->nexthops[0].ifindex;
6157 	struct mlxsw_sp_ipip_entry *ipip_entry;
6158 
6159 	switch (fen_info->type) {
6160 	case RTN_LOCAL:
6161 		ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
6162 							       MLXSW_SP_L3_PROTO_IPV4, dip);
6163 		if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
6164 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
6165 			return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
6166 							     fib_entry,
6167 							     ipip_entry);
6168 		}
6169 		if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
6170 						 MLXSW_SP_L3_PROTO_IPV4,
6171 						 &dip)) {
6172 			u32 tunnel_index;
6173 
6174 			tunnel_index = router->nve_decap_config.tunnel_index;
6175 			fib_entry->decap.tunnel_index = tunnel_index;
6176 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
6177 			return 0;
6178 		}
6179 		fallthrough;
6180 	case RTN_BROADCAST:
6181 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
6182 		return 0;
6183 	case RTN_BLACKHOLE:
6184 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
6185 		return 0;
6186 	case RTN_UNREACHABLE:
6187 	case RTN_PROHIBIT:
6188 		/* Packets hitting these routes need to be trapped, but
6189 		 * can do so with a lower priority than packets directed
6190 		 * at the host, so use action type local instead of trap.
6191 		 */
6192 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
6193 		return 0;
6194 	case RTN_UNICAST:
6195 		if (nhgi->gateway)
6196 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
6197 		else
6198 			fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
6199 		return 0;
6200 	default:
6201 		return -EINVAL;
6202 	}
6203 }
6204 
6205 static void
6206 mlxsw_sp_fib_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6207 			      struct mlxsw_sp_fib_entry *fib_entry)
6208 {
6209 	switch (fib_entry->type) {
6210 	case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
6211 		mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
6212 		break;
6213 	default:
6214 		break;
6215 	}
6216 }
6217 
6218 static void
6219 mlxsw_sp_fib4_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
6220 			       struct mlxsw_sp_fib4_entry *fib4_entry)
6221 {
6222 	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib4_entry->common);
6223 }
6224 
6225 static struct mlxsw_sp_fib4_entry *
6226 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
6227 			   struct mlxsw_sp_fib_node *fib_node,
6228 			   const struct fib_entry_notifier_info *fen_info)
6229 {
6230 	struct mlxsw_sp_fib4_entry *fib4_entry;
6231 	struct mlxsw_sp_fib_entry *fib_entry;
6232 	int err;
6233 
6234 	fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
6235 	if (!fib4_entry)
6236 		return ERR_PTR(-ENOMEM);
6237 	fib_entry = &fib4_entry->common;
6238 
6239 	err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
6240 	if (err)
6241 		goto err_nexthop4_group_get;
6242 
6243 	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
6244 					     fib_node->fib);
6245 	if (err)
6246 		goto err_nexthop_group_vr_link;
6247 
6248 	err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
6249 	if (err)
6250 		goto err_fib4_entry_type_set;
6251 
6252 	fib4_entry->fi = fen_info->fi;
6253 	fib_info_hold(fib4_entry->fi);
6254 	fib4_entry->tb_id = fen_info->tb_id;
6255 	fib4_entry->type = fen_info->type;
6256 	fib4_entry->dscp = fen_info->dscp;
6257 
6258 	fib_entry->fib_node = fib_node;
6259 
6260 	return fib4_entry;
6261 
6262 err_fib4_entry_type_set:
6263 	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
6264 err_nexthop_group_vr_link:
6265 	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6266 err_nexthop4_group_get:
6267 	kfree(fib4_entry);
6268 	return ERR_PTR(err);
6269 }
6270 
6271 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
6272 					struct mlxsw_sp_fib4_entry *fib4_entry)
6273 {
6274 	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6275 
6276 	fib_info_put(fib4_entry->fi);
6277 	mlxsw_sp_fib4_entry_type_unset(mlxsw_sp, fib4_entry);
6278 	mlxsw_sp_nexthop_group_vr_unlink(fib4_entry->common.nh_group,
6279 					 fib_node->fib);
6280 	mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
6281 	kfree(fib4_entry);
6282 }
6283 
6284 static struct mlxsw_sp_fib4_entry *
6285 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
6286 			   const struct fib_entry_notifier_info *fen_info)
6287 {
6288 	struct mlxsw_sp_fib4_entry *fib4_entry;
6289 	struct mlxsw_sp_fib_node *fib_node;
6290 	struct mlxsw_sp_fib *fib;
6291 	struct mlxsw_sp_vr *vr;
6292 
6293 	vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
6294 	if (!vr)
6295 		return NULL;
6296 	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
6297 
6298 	fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
6299 					    sizeof(fen_info->dst),
6300 					    fen_info->dst_len);
6301 	if (!fib_node)
6302 		return NULL;
6303 
6304 	fib4_entry = container_of(fib_node->fib_entry,
6305 				  struct mlxsw_sp_fib4_entry, common);
6306 	if (fib4_entry->tb_id == fen_info->tb_id &&
6307 	    fib4_entry->dscp == fen_info->dscp &&
6308 	    fib4_entry->type == fen_info->type &&
6309 	    fib4_entry->fi == fen_info->fi)
6310 		return fib4_entry;
6311 
6312 	return NULL;
6313 }
6314 
6315 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
6316 	.key_offset = offsetof(struct mlxsw_sp_fib_node, key),
6317 	.head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
6318 	.key_len = sizeof(struct mlxsw_sp_fib_key),
6319 	.automatic_shrinking = true,
6320 };
6321 
6322 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
6323 				    struct mlxsw_sp_fib_node *fib_node)
6324 {
6325 	return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
6326 				      mlxsw_sp_fib_ht_params);
6327 }
6328 
6329 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
6330 				     struct mlxsw_sp_fib_node *fib_node)
6331 {
6332 	rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
6333 			       mlxsw_sp_fib_ht_params);
6334 }
6335 
6336 static struct mlxsw_sp_fib_node *
6337 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
6338 			 size_t addr_len, unsigned char prefix_len)
6339 {
6340 	struct mlxsw_sp_fib_key key;
6341 
6342 	memset(&key, 0, sizeof(key));
6343 	memcpy(key.addr, addr, addr_len);
6344 	key.prefix_len = prefix_len;
6345 	return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
6346 }
6347 
6348 static struct mlxsw_sp_fib_node *
6349 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
6350 			 size_t addr_len, unsigned char prefix_len)
6351 {
6352 	struct mlxsw_sp_fib_node *fib_node;
6353 
6354 	fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
6355 	if (!fib_node)
6356 		return NULL;
6357 
6358 	list_add(&fib_node->list, &fib->node_list);
6359 	memcpy(fib_node->key.addr, addr, addr_len);
6360 	fib_node->key.prefix_len = prefix_len;
6361 
6362 	return fib_node;
6363 }
6364 
6365 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
6366 {
6367 	list_del(&fib_node->list);
6368 	kfree(fib_node);
6369 }
6370 
6371 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
6372 				      struct mlxsw_sp_fib_node *fib_node)
6373 {
6374 	struct mlxsw_sp_prefix_usage req_prefix_usage;
6375 	struct mlxsw_sp_fib *fib = fib_node->fib;
6376 	struct mlxsw_sp_lpm_tree *lpm_tree;
6377 	int err;
6378 
6379 	lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
6380 	if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6381 		goto out;
6382 
6383 	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6384 	mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
6385 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6386 					 fib->proto);
6387 	if (IS_ERR(lpm_tree))
6388 		return PTR_ERR(lpm_tree);
6389 
6390 	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6391 	if (err)
6392 		goto err_lpm_tree_replace;
6393 
6394 out:
6395 	lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
6396 	return 0;
6397 
6398 err_lpm_tree_replace:
6399 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6400 	return err;
6401 }
6402 
6403 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
6404 					 struct mlxsw_sp_fib_node *fib_node)
6405 {
6406 	struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
6407 	struct mlxsw_sp_prefix_usage req_prefix_usage;
6408 	struct mlxsw_sp_fib *fib = fib_node->fib;
6409 	int err;
6410 
6411 	if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
6412 		return;
6413 	/* Try to construct a new LPM tree from the current prefix usage
6414 	 * minus the unused one. If we fail, continue using the old one.
6415 	 */
6416 	mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
6417 	mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
6418 				    fib_node->key.prefix_len);
6419 	lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
6420 					 fib->proto);
6421 	if (IS_ERR(lpm_tree))
6422 		return;
6423 
6424 	err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
6425 	if (err)
6426 		goto err_lpm_tree_replace;
6427 
6428 	return;
6429 
6430 err_lpm_tree_replace:
6431 	mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
6432 }
6433 
6434 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
6435 				  struct mlxsw_sp_fib_node *fib_node,
6436 				  struct mlxsw_sp_fib *fib)
6437 {
6438 	int err;
6439 
6440 	err = mlxsw_sp_fib_node_insert(fib, fib_node);
6441 	if (err)
6442 		return err;
6443 	fib_node->fib = fib;
6444 
6445 	err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
6446 	if (err)
6447 		goto err_fib_lpm_tree_link;
6448 
6449 	return 0;
6450 
6451 err_fib_lpm_tree_link:
6452 	fib_node->fib = NULL;
6453 	mlxsw_sp_fib_node_remove(fib, fib_node);
6454 	return err;
6455 }
6456 
6457 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
6458 				   struct mlxsw_sp_fib_node *fib_node)
6459 {
6460 	struct mlxsw_sp_fib *fib = fib_node->fib;
6461 
6462 	mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
6463 	fib_node->fib = NULL;
6464 	mlxsw_sp_fib_node_remove(fib, fib_node);
6465 }
6466 
6467 static struct mlxsw_sp_fib_node *
6468 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
6469 		      size_t addr_len, unsigned char prefix_len,
6470 		      enum mlxsw_sp_l3proto proto)
6471 {
6472 	struct mlxsw_sp_fib_node *fib_node;
6473 	struct mlxsw_sp_fib *fib;
6474 	struct mlxsw_sp_vr *vr;
6475 	int err;
6476 
6477 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
6478 	if (IS_ERR(vr))
6479 		return ERR_CAST(vr);
6480 	fib = mlxsw_sp_vr_fib(vr, proto);
6481 
6482 	fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
6483 	if (fib_node)
6484 		return fib_node;
6485 
6486 	fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
6487 	if (!fib_node) {
6488 		err = -ENOMEM;
6489 		goto err_fib_node_create;
6490 	}
6491 
6492 	err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
6493 	if (err)
6494 		goto err_fib_node_init;
6495 
6496 	return fib_node;
6497 
6498 err_fib_node_init:
6499 	mlxsw_sp_fib_node_destroy(fib_node);
6500 err_fib_node_create:
6501 	mlxsw_sp_vr_put(mlxsw_sp, vr);
6502 	return ERR_PTR(err);
6503 }
6504 
6505 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
6506 				  struct mlxsw_sp_fib_node *fib_node)
6507 {
6508 	struct mlxsw_sp_vr *vr = fib_node->fib->vr;
6509 
6510 	if (fib_node->fib_entry)
6511 		return;
6512 	mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
6513 	mlxsw_sp_fib_node_destroy(fib_node);
6514 	mlxsw_sp_vr_put(mlxsw_sp, vr);
6515 }
6516 
6517 static int mlxsw_sp_fib_node_entry_link(struct mlxsw_sp *mlxsw_sp,
6518 					struct mlxsw_sp_fib_entry *fib_entry)
6519 {
6520 	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6521 	int err;
6522 
6523 	fib_node->fib_entry = fib_entry;
6524 
6525 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
6526 	if (err)
6527 		goto err_fib_entry_update;
6528 
6529 	return 0;
6530 
6531 err_fib_entry_update:
6532 	fib_node->fib_entry = NULL;
6533 	return err;
6534 }
6535 
6536 static void
6537 mlxsw_sp_fib_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
6538 			       struct mlxsw_sp_fib_entry *fib_entry)
6539 {
6540 	struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
6541 
6542 	mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
6543 	fib_node->fib_entry = NULL;
6544 }
6545 
6546 static bool mlxsw_sp_fib4_allow_replace(struct mlxsw_sp_fib4_entry *fib4_entry)
6547 {
6548 	struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
6549 	struct mlxsw_sp_fib4_entry *fib4_replaced;
6550 
6551 	if (!fib_node->fib_entry)
6552 		return true;
6553 
6554 	fib4_replaced = container_of(fib_node->fib_entry,
6555 				     struct mlxsw_sp_fib4_entry, common);
6556 	if (fib4_entry->tb_id == RT_TABLE_MAIN &&
6557 	    fib4_replaced->tb_id == RT_TABLE_LOCAL)
6558 		return false;
6559 
6560 	return true;
6561 }
6562 
6563 static int
6564 mlxsw_sp_router_fib4_replace(struct mlxsw_sp *mlxsw_sp,
6565 			     const struct fib_entry_notifier_info *fen_info)
6566 {
6567 	struct mlxsw_sp_fib4_entry *fib4_entry, *fib4_replaced;
6568 	struct mlxsw_sp_fib_entry *replaced;
6569 	struct mlxsw_sp_fib_node *fib_node;
6570 	int err;
6571 
6572 	if (fen_info->fi->nh &&
6573 	    !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, fen_info->fi->nh->id))
6574 		return 0;
6575 
6576 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
6577 					 &fen_info->dst, sizeof(fen_info->dst),
6578 					 fen_info->dst_len,
6579 					 MLXSW_SP_L3_PROTO_IPV4);
6580 	if (IS_ERR(fib_node)) {
6581 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
6582 		return PTR_ERR(fib_node);
6583 	}
6584 
6585 	fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
6586 	if (IS_ERR(fib4_entry)) {
6587 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
6588 		err = PTR_ERR(fib4_entry);
6589 		goto err_fib4_entry_create;
6590 	}
6591 
6592 	if (!mlxsw_sp_fib4_allow_replace(fib4_entry)) {
6593 		mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6594 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6595 		return 0;
6596 	}
6597 
6598 	replaced = fib_node->fib_entry;
6599 	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib4_entry->common);
6600 	if (err) {
6601 		dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
6602 		goto err_fib_node_entry_link;
6603 	}
6604 
6605 	/* Nothing to replace */
6606 	if (!replaced)
6607 		return 0;
6608 
6609 	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
6610 	fib4_replaced = container_of(replaced, struct mlxsw_sp_fib4_entry,
6611 				     common);
6612 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_replaced);
6613 
6614 	return 0;
6615 
6616 err_fib_node_entry_link:
6617 	fib_node->fib_entry = replaced;
6618 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6619 err_fib4_entry_create:
6620 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6621 	return err;
6622 }
6623 
6624 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
6625 				     struct fib_entry_notifier_info *fen_info)
6626 {
6627 	struct mlxsw_sp_fib4_entry *fib4_entry;
6628 	struct mlxsw_sp_fib_node *fib_node;
6629 
6630 	fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
6631 	if (!fib4_entry)
6632 		return;
6633 	fib_node = fib4_entry->common.fib_node;
6634 
6635 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib4_entry->common);
6636 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
6637 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
6638 }
6639 
6640 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
6641 {
6642 	/* Multicast routes aren't supported, so ignore them. Neighbour
6643 	 * Discovery packets are specifically trapped.
6644 	 */
6645 	if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
6646 		return true;
6647 
6648 	/* Cloned routes are irrelevant in the forwarding path. */
6649 	if (rt->fib6_flags & RTF_CACHE)
6650 		return true;
6651 
6652 	return false;
6653 }
6654 
6655 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
6656 {
6657 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6658 
6659 	mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
6660 	if (!mlxsw_sp_rt6)
6661 		return ERR_PTR(-ENOMEM);
6662 
6663 	/* In case of route replace, replaced route is deleted with
6664 	 * no notification. Take reference to prevent accessing freed
6665 	 * memory.
6666 	 */
6667 	mlxsw_sp_rt6->rt = rt;
6668 	fib6_info_hold(rt);
6669 
6670 	return mlxsw_sp_rt6;
6671 }
6672 
6673 #if IS_ENABLED(CONFIG_IPV6)
6674 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6675 {
6676 	fib6_info_release(rt);
6677 }
6678 #else
6679 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
6680 {
6681 }
6682 #endif
6683 
6684 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
6685 {
6686 	struct fib6_nh *fib6_nh = mlxsw_sp_rt6->rt->fib6_nh;
6687 
6688 	if (!mlxsw_sp_rt6->rt->nh)
6689 		fib6_nh->fib_nh_flags &= ~RTNH_F_OFFLOAD;
6690 	mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
6691 	kfree(mlxsw_sp_rt6);
6692 }
6693 
6694 static struct fib6_info *
6695 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
6696 {
6697 	return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
6698 				list)->rt;
6699 }
6700 
6701 static struct mlxsw_sp_rt6 *
6702 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
6703 			    const struct fib6_info *rt)
6704 {
6705 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6706 
6707 	list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
6708 		if (mlxsw_sp_rt6->rt == rt)
6709 			return mlxsw_sp_rt6;
6710 	}
6711 
6712 	return NULL;
6713 }
6714 
6715 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
6716 					const struct fib6_info *rt,
6717 					enum mlxsw_sp_ipip_type *ret)
6718 {
6719 	return rt->fib6_nh->fib_nh_dev &&
6720 	       mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh->fib_nh_dev, ret);
6721 }
6722 
6723 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
6724 				  struct mlxsw_sp_nexthop_group *nh_grp,
6725 				  struct mlxsw_sp_nexthop *nh,
6726 				  const struct fib6_info *rt)
6727 {
6728 	struct net_device *dev = rt->fib6_nh->fib_nh_dev;
6729 	int err;
6730 
6731 	nh->nhgi = nh_grp->nhgi;
6732 	nh->nh_weight = rt->fib6_nh->fib_nh_weight;
6733 	memcpy(&nh->gw_addr, &rt->fib6_nh->fib_nh_gw6, sizeof(nh->gw_addr));
6734 #if IS_ENABLED(CONFIG_IPV6)
6735 	nh->neigh_tbl = &nd_tbl;
6736 #endif
6737 	mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
6738 
6739 	list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
6740 
6741 	if (!dev)
6742 		return 0;
6743 	nh->ifindex = dev->ifindex;
6744 
6745 	err = mlxsw_sp_nexthop_type_init(mlxsw_sp, nh, dev);
6746 	if (err)
6747 		goto err_nexthop_type_init;
6748 
6749 	return 0;
6750 
6751 err_nexthop_type_init:
6752 	list_del(&nh->router_list_node);
6753 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6754 	return err;
6755 }
6756 
6757 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
6758 				   struct mlxsw_sp_nexthop *nh)
6759 {
6760 	mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
6761 	list_del(&nh->router_list_node);
6762 	mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
6763 }
6764 
6765 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
6766 				    const struct fib6_info *rt)
6767 {
6768 	return rt->fib6_nh->fib_nh_gw_family ||
6769 	       mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
6770 }
6771 
6772 static int
6773 mlxsw_sp_nexthop6_group_info_init(struct mlxsw_sp *mlxsw_sp,
6774 				  struct mlxsw_sp_nexthop_group *nh_grp,
6775 				  struct mlxsw_sp_fib6_entry *fib6_entry)
6776 {
6777 	struct mlxsw_sp_nexthop_group_info *nhgi;
6778 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6779 	struct mlxsw_sp_nexthop *nh;
6780 	int err, i;
6781 
6782 	nhgi = kzalloc(struct_size(nhgi, nexthops, fib6_entry->nrt6),
6783 		       GFP_KERNEL);
6784 	if (!nhgi)
6785 		return -ENOMEM;
6786 	nh_grp->nhgi = nhgi;
6787 	nhgi->nh_grp = nh_grp;
6788 	mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
6789 					struct mlxsw_sp_rt6, list);
6790 	nhgi->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
6791 	nhgi->count = fib6_entry->nrt6;
6792 	for (i = 0; i < nhgi->count; i++) {
6793 		struct fib6_info *rt = mlxsw_sp_rt6->rt;
6794 
6795 		nh = &nhgi->nexthops[i];
6796 		err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
6797 		if (err)
6798 			goto err_nexthop6_init;
6799 		mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
6800 	}
6801 	nh_grp->nhgi = nhgi;
6802 	err = mlxsw_sp_nexthop_group_inc(mlxsw_sp);
6803 	if (err)
6804 		goto err_group_inc;
6805 	err = mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6806 	if (err)
6807 		goto err_group_refresh;
6808 
6809 	return 0;
6810 
6811 err_group_refresh:
6812 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6813 err_group_inc:
6814 	i = nhgi->count;
6815 err_nexthop6_init:
6816 	for (i--; i >= 0; i--) {
6817 		nh = &nhgi->nexthops[i];
6818 		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6819 	}
6820 	kfree(nhgi);
6821 	return err;
6822 }
6823 
6824 static void
6825 mlxsw_sp_nexthop6_group_info_fini(struct mlxsw_sp *mlxsw_sp,
6826 				  struct mlxsw_sp_nexthop_group *nh_grp)
6827 {
6828 	struct mlxsw_sp_nexthop_group_info *nhgi = nh_grp->nhgi;
6829 	int i;
6830 
6831 	mlxsw_sp_nexthop_group_dec(mlxsw_sp);
6832 	for (i = nhgi->count - 1; i >= 0; i--) {
6833 		struct mlxsw_sp_nexthop *nh = &nhgi->nexthops[i];
6834 
6835 		mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
6836 	}
6837 	mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
6838 	WARN_ON_ONCE(nhgi->adj_index_valid);
6839 	kfree(nhgi);
6840 }
6841 
6842 static struct mlxsw_sp_nexthop_group *
6843 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
6844 			       struct mlxsw_sp_fib6_entry *fib6_entry)
6845 {
6846 	struct mlxsw_sp_nexthop_group *nh_grp;
6847 	int err;
6848 
6849 	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
6850 	if (!nh_grp)
6851 		return ERR_PTR(-ENOMEM);
6852 	INIT_LIST_HEAD(&nh_grp->vr_list);
6853 	err = rhashtable_init(&nh_grp->vr_ht,
6854 			      &mlxsw_sp_nexthop_group_vr_ht_params);
6855 	if (err)
6856 		goto err_nexthop_group_vr_ht_init;
6857 	INIT_LIST_HEAD(&nh_grp->fib_list);
6858 	nh_grp->type = MLXSW_SP_NEXTHOP_GROUP_TYPE_IPV6;
6859 
6860 	err = mlxsw_sp_nexthop6_group_info_init(mlxsw_sp, nh_grp, fib6_entry);
6861 	if (err)
6862 		goto err_nexthop_group_info_init;
6863 
6864 	err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
6865 	if (err)
6866 		goto err_nexthop_group_insert;
6867 
6868 	nh_grp->can_destroy = true;
6869 
6870 	return nh_grp;
6871 
6872 err_nexthop_group_insert:
6873 	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6874 err_nexthop_group_info_init:
6875 	rhashtable_destroy(&nh_grp->vr_ht);
6876 err_nexthop_group_vr_ht_init:
6877 	kfree(nh_grp);
6878 	return ERR_PTR(err);
6879 }
6880 
6881 static void
6882 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
6883 				struct mlxsw_sp_nexthop_group *nh_grp)
6884 {
6885 	if (!nh_grp->can_destroy)
6886 		return;
6887 	mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
6888 	mlxsw_sp_nexthop6_group_info_fini(mlxsw_sp, nh_grp);
6889 	WARN_ON_ONCE(!list_empty(&nh_grp->vr_list));
6890 	rhashtable_destroy(&nh_grp->vr_ht);
6891 	kfree(nh_grp);
6892 }
6893 
6894 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
6895 				       struct mlxsw_sp_fib6_entry *fib6_entry)
6896 {
6897 	struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
6898 	struct mlxsw_sp_nexthop_group *nh_grp;
6899 
6900 	if (rt->nh) {
6901 		nh_grp = mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp,
6902 							   rt->nh->id);
6903 		if (WARN_ON_ONCE(!nh_grp))
6904 			return -EINVAL;
6905 		goto out;
6906 	}
6907 
6908 	nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
6909 	if (!nh_grp) {
6910 		nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
6911 		if (IS_ERR(nh_grp))
6912 			return PTR_ERR(nh_grp);
6913 	}
6914 
6915 	/* The route and the nexthop are described by the same struct, so we
6916 	 * need to the update the nexthop offload indication for the new route.
6917 	 */
6918 	__mlxsw_sp_nexthop6_group_offload_refresh(nh_grp, fib6_entry);
6919 
6920 out:
6921 	list_add_tail(&fib6_entry->common.nexthop_group_node,
6922 		      &nh_grp->fib_list);
6923 	fib6_entry->common.nh_group = nh_grp;
6924 
6925 	return 0;
6926 }
6927 
6928 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
6929 					struct mlxsw_sp_fib_entry *fib_entry)
6930 {
6931 	struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
6932 
6933 	list_del(&fib_entry->nexthop_group_node);
6934 	if (!list_empty(&nh_grp->fib_list))
6935 		return;
6936 
6937 	if (nh_grp->type == MLXSW_SP_NEXTHOP_GROUP_TYPE_OBJ) {
6938 		mlxsw_sp_nexthop_obj_group_destroy(mlxsw_sp, nh_grp);
6939 		return;
6940 	}
6941 
6942 	mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
6943 }
6944 
6945 static int
6946 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
6947 			       struct mlxsw_sp_fib6_entry *fib6_entry)
6948 {
6949 	struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
6950 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
6951 	int err;
6952 
6953 	mlxsw_sp_nexthop_group_vr_unlink(old_nh_grp, fib_node->fib);
6954 	fib6_entry->common.nh_group = NULL;
6955 	list_del(&fib6_entry->common.nexthop_group_node);
6956 
6957 	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
6958 	if (err)
6959 		goto err_nexthop6_group_get;
6960 
6961 	err = mlxsw_sp_nexthop_group_vr_link(fib6_entry->common.nh_group,
6962 					     fib_node->fib);
6963 	if (err)
6964 		goto err_nexthop_group_vr_link;
6965 
6966 	/* In case this entry is offloaded, then the adjacency index
6967 	 * currently associated with it in the device's table is that
6968 	 * of the old group. Start using the new one instead.
6969 	 */
6970 	err = mlxsw_sp_fib_entry_update(mlxsw_sp, &fib6_entry->common);
6971 	if (err)
6972 		goto err_fib_entry_update;
6973 
6974 	if (list_empty(&old_nh_grp->fib_list))
6975 		mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
6976 
6977 	return 0;
6978 
6979 err_fib_entry_update:
6980 	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
6981 					 fib_node->fib);
6982 err_nexthop_group_vr_link:
6983 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
6984 err_nexthop6_group_get:
6985 	list_add_tail(&fib6_entry->common.nexthop_group_node,
6986 		      &old_nh_grp->fib_list);
6987 	fib6_entry->common.nh_group = old_nh_grp;
6988 	mlxsw_sp_nexthop_group_vr_link(old_nh_grp, fib_node->fib);
6989 	return err;
6990 }
6991 
6992 static int
6993 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
6994 				struct mlxsw_sp_fib6_entry *fib6_entry,
6995 				struct fib6_info **rt_arr, unsigned int nrt6)
6996 {
6997 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
6998 	int err, i;
6999 
7000 	for (i = 0; i < nrt6; i++) {
7001 		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
7002 		if (IS_ERR(mlxsw_sp_rt6)) {
7003 			err = PTR_ERR(mlxsw_sp_rt6);
7004 			goto err_rt6_unwind;
7005 		}
7006 
7007 		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
7008 		fib6_entry->nrt6++;
7009 	}
7010 
7011 	err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7012 	if (err)
7013 		goto err_rt6_unwind;
7014 
7015 	return 0;
7016 
7017 err_rt6_unwind:
7018 	for (; i > 0; i--) {
7019 		fib6_entry->nrt6--;
7020 		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7021 					       struct mlxsw_sp_rt6, list);
7022 		list_del(&mlxsw_sp_rt6->list);
7023 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7024 	}
7025 	return err;
7026 }
7027 
7028 static void
7029 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
7030 				struct mlxsw_sp_fib6_entry *fib6_entry,
7031 				struct fib6_info **rt_arr, unsigned int nrt6)
7032 {
7033 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7034 	int i;
7035 
7036 	for (i = 0; i < nrt6; i++) {
7037 		mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry,
7038 							   rt_arr[i]);
7039 		if (WARN_ON_ONCE(!mlxsw_sp_rt6))
7040 			continue;
7041 
7042 		fib6_entry->nrt6--;
7043 		list_del(&mlxsw_sp_rt6->list);
7044 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7045 	}
7046 
7047 	mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
7048 }
7049 
7050 static int
7051 mlxsw_sp_fib6_entry_type_set_local(struct mlxsw_sp *mlxsw_sp,
7052 				   struct mlxsw_sp_fib_entry *fib_entry,
7053 				   const struct fib6_info *rt)
7054 {
7055 	struct mlxsw_sp_nexthop_group_info *nhgi = fib_entry->nh_group->nhgi;
7056 	union mlxsw_sp_l3addr dip = { .addr6 = rt->fib6_dst.addr };
7057 	u32 tb_id = mlxsw_sp_fix_tb_id(rt->fib6_table->tb6_id);
7058 	struct mlxsw_sp_router *router = mlxsw_sp->router;
7059 	int ifindex = nhgi->nexthops[0].ifindex;
7060 	struct mlxsw_sp_ipip_entry *ipip_entry;
7061 
7062 	fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7063 	ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, ifindex,
7064 						       MLXSW_SP_L3_PROTO_IPV6,
7065 						       dip);
7066 
7067 	if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
7068 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
7069 		return mlxsw_sp_fib_entry_decap_init(mlxsw_sp, fib_entry,
7070 						     ipip_entry);
7071 	}
7072 	if (mlxsw_sp_router_nve_is_decap(mlxsw_sp, tb_id,
7073 					 MLXSW_SP_L3_PROTO_IPV6, &dip)) {
7074 		u32 tunnel_index;
7075 
7076 		tunnel_index = router->nve_decap_config.tunnel_index;
7077 		fib_entry->decap.tunnel_index = tunnel_index;
7078 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
7079 	}
7080 
7081 	return 0;
7082 }
7083 
7084 static int mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
7085 					struct mlxsw_sp_fib_entry *fib_entry,
7086 					const struct fib6_info *rt)
7087 {
7088 	if (rt->fib6_flags & RTF_LOCAL)
7089 		return mlxsw_sp_fib6_entry_type_set_local(mlxsw_sp, fib_entry,
7090 							  rt);
7091 	if (rt->fib6_flags & RTF_ANYCAST)
7092 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
7093 	else if (rt->fib6_type == RTN_BLACKHOLE)
7094 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
7095 	else if (rt->fib6_flags & RTF_REJECT)
7096 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_UNREACHABLE;
7097 	else if (fib_entry->nh_group->nhgi->gateway)
7098 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
7099 	else
7100 		fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
7101 
7102 	return 0;
7103 }
7104 
7105 static void
7106 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
7107 {
7108 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
7109 
7110 	list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
7111 				 list) {
7112 		fib6_entry->nrt6--;
7113 		list_del(&mlxsw_sp_rt6->list);
7114 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7115 	}
7116 }
7117 
7118 static struct mlxsw_sp_fib6_entry *
7119 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
7120 			   struct mlxsw_sp_fib_node *fib_node,
7121 			   struct fib6_info **rt_arr, unsigned int nrt6)
7122 {
7123 	struct mlxsw_sp_fib6_entry *fib6_entry;
7124 	struct mlxsw_sp_fib_entry *fib_entry;
7125 	struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
7126 	int err, i;
7127 
7128 	fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
7129 	if (!fib6_entry)
7130 		return ERR_PTR(-ENOMEM);
7131 	fib_entry = &fib6_entry->common;
7132 
7133 	INIT_LIST_HEAD(&fib6_entry->rt6_list);
7134 
7135 	for (i = 0; i < nrt6; i++) {
7136 		mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt_arr[i]);
7137 		if (IS_ERR(mlxsw_sp_rt6)) {
7138 			err = PTR_ERR(mlxsw_sp_rt6);
7139 			goto err_rt6_unwind;
7140 		}
7141 		list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
7142 		fib6_entry->nrt6++;
7143 	}
7144 
7145 	err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
7146 	if (err)
7147 		goto err_rt6_unwind;
7148 
7149 	err = mlxsw_sp_nexthop_group_vr_link(fib_entry->nh_group,
7150 					     fib_node->fib);
7151 	if (err)
7152 		goto err_nexthop_group_vr_link;
7153 
7154 	err = mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, rt_arr[0]);
7155 	if (err)
7156 		goto err_fib6_entry_type_set;
7157 
7158 	fib_entry->fib_node = fib_node;
7159 
7160 	return fib6_entry;
7161 
7162 err_fib6_entry_type_set:
7163 	mlxsw_sp_nexthop_group_vr_unlink(fib_entry->nh_group, fib_node->fib);
7164 err_nexthop_group_vr_link:
7165 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, fib_entry);
7166 err_rt6_unwind:
7167 	for (; i > 0; i--) {
7168 		fib6_entry->nrt6--;
7169 		mlxsw_sp_rt6 = list_last_entry(&fib6_entry->rt6_list,
7170 					       struct mlxsw_sp_rt6, list);
7171 		list_del(&mlxsw_sp_rt6->list);
7172 		mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
7173 	}
7174 	kfree(fib6_entry);
7175 	return ERR_PTR(err);
7176 }
7177 
7178 static void
7179 mlxsw_sp_fib6_entry_type_unset(struct mlxsw_sp *mlxsw_sp,
7180 			       struct mlxsw_sp_fib6_entry *fib6_entry)
7181 {
7182 	mlxsw_sp_fib_entry_type_unset(mlxsw_sp, &fib6_entry->common);
7183 }
7184 
7185 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
7186 					struct mlxsw_sp_fib6_entry *fib6_entry)
7187 {
7188 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7189 
7190 	mlxsw_sp_fib6_entry_type_unset(mlxsw_sp, fib6_entry);
7191 	mlxsw_sp_nexthop_group_vr_unlink(fib6_entry->common.nh_group,
7192 					 fib_node->fib);
7193 	mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
7194 	mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
7195 	WARN_ON(fib6_entry->nrt6);
7196 	kfree(fib6_entry);
7197 }
7198 
7199 static struct mlxsw_sp_fib6_entry *
7200 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
7201 			   const struct fib6_info *rt)
7202 {
7203 	struct mlxsw_sp_fib6_entry *fib6_entry;
7204 	struct mlxsw_sp_fib_node *fib_node;
7205 	struct mlxsw_sp_fib *fib;
7206 	struct fib6_info *cmp_rt;
7207 	struct mlxsw_sp_vr *vr;
7208 
7209 	vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
7210 	if (!vr)
7211 		return NULL;
7212 	fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
7213 
7214 	fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
7215 					    sizeof(rt->fib6_dst.addr),
7216 					    rt->fib6_dst.plen);
7217 	if (!fib_node)
7218 		return NULL;
7219 
7220 	fib6_entry = container_of(fib_node->fib_entry,
7221 				  struct mlxsw_sp_fib6_entry, common);
7222 	cmp_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7223 	if (rt->fib6_table->tb6_id == cmp_rt->fib6_table->tb6_id &&
7224 	    rt->fib6_metric == cmp_rt->fib6_metric &&
7225 	    mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
7226 		return fib6_entry;
7227 
7228 	return NULL;
7229 }
7230 
7231 static bool mlxsw_sp_fib6_allow_replace(struct mlxsw_sp_fib6_entry *fib6_entry)
7232 {
7233 	struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
7234 	struct mlxsw_sp_fib6_entry *fib6_replaced;
7235 	struct fib6_info *rt, *rt_replaced;
7236 
7237 	if (!fib_node->fib_entry)
7238 		return true;
7239 
7240 	fib6_replaced = container_of(fib_node->fib_entry,
7241 				     struct mlxsw_sp_fib6_entry,
7242 				     common);
7243 	rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
7244 	rt_replaced = mlxsw_sp_fib6_entry_rt(fib6_replaced);
7245 	if (rt->fib6_table->tb6_id == RT_TABLE_MAIN &&
7246 	    rt_replaced->fib6_table->tb6_id == RT_TABLE_LOCAL)
7247 		return false;
7248 
7249 	return true;
7250 }
7251 
7252 static int mlxsw_sp_router_fib6_replace(struct mlxsw_sp *mlxsw_sp,
7253 					struct fib6_info **rt_arr,
7254 					unsigned int nrt6)
7255 {
7256 	struct mlxsw_sp_fib6_entry *fib6_entry, *fib6_replaced;
7257 	struct mlxsw_sp_fib_entry *replaced;
7258 	struct mlxsw_sp_fib_node *fib_node;
7259 	struct fib6_info *rt = rt_arr[0];
7260 	int err;
7261 
7262 	if (rt->fib6_src.plen)
7263 		return -EINVAL;
7264 
7265 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7266 		return 0;
7267 
7268 	if (rt->nh && !mlxsw_sp_nexthop_obj_group_lookup(mlxsw_sp, rt->nh->id))
7269 		return 0;
7270 
7271 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7272 					 &rt->fib6_dst.addr,
7273 					 sizeof(rt->fib6_dst.addr),
7274 					 rt->fib6_dst.plen,
7275 					 MLXSW_SP_L3_PROTO_IPV6);
7276 	if (IS_ERR(fib_node))
7277 		return PTR_ERR(fib_node);
7278 
7279 	fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt_arr,
7280 						nrt6);
7281 	if (IS_ERR(fib6_entry)) {
7282 		err = PTR_ERR(fib6_entry);
7283 		goto err_fib6_entry_create;
7284 	}
7285 
7286 	if (!mlxsw_sp_fib6_allow_replace(fib6_entry)) {
7287 		mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7288 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7289 		return 0;
7290 	}
7291 
7292 	replaced = fib_node->fib_entry;
7293 	err = mlxsw_sp_fib_node_entry_link(mlxsw_sp, &fib6_entry->common);
7294 	if (err)
7295 		goto err_fib_node_entry_link;
7296 
7297 	/* Nothing to replace */
7298 	if (!replaced)
7299 		return 0;
7300 
7301 	mlxsw_sp_fib_entry_hw_flags_clear(mlxsw_sp, replaced);
7302 	fib6_replaced = container_of(replaced, struct mlxsw_sp_fib6_entry,
7303 				     common);
7304 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_replaced);
7305 
7306 	return 0;
7307 
7308 err_fib_node_entry_link:
7309 	fib_node->fib_entry = replaced;
7310 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7311 err_fib6_entry_create:
7312 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7313 	return err;
7314 }
7315 
7316 static int mlxsw_sp_router_fib6_append(struct mlxsw_sp *mlxsw_sp,
7317 				       struct fib6_info **rt_arr,
7318 				       unsigned int nrt6)
7319 {
7320 	struct mlxsw_sp_fib6_entry *fib6_entry;
7321 	struct mlxsw_sp_fib_node *fib_node;
7322 	struct fib6_info *rt = rt_arr[0];
7323 	int err;
7324 
7325 	if (rt->fib6_src.plen)
7326 		return -EINVAL;
7327 
7328 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7329 		return 0;
7330 
7331 	fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
7332 					 &rt->fib6_dst.addr,
7333 					 sizeof(rt->fib6_dst.addr),
7334 					 rt->fib6_dst.plen,
7335 					 MLXSW_SP_L3_PROTO_IPV6);
7336 	if (IS_ERR(fib_node))
7337 		return PTR_ERR(fib_node);
7338 
7339 	if (WARN_ON_ONCE(!fib_node->fib_entry)) {
7340 		mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7341 		return -EINVAL;
7342 	}
7343 
7344 	fib6_entry = container_of(fib_node->fib_entry,
7345 				  struct mlxsw_sp_fib6_entry, common);
7346 	err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt_arr,
7347 					      nrt6);
7348 	if (err)
7349 		goto err_fib6_entry_nexthop_add;
7350 
7351 	return 0;
7352 
7353 err_fib6_entry_nexthop_add:
7354 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7355 	return err;
7356 }
7357 
7358 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
7359 				     struct fib6_info **rt_arr,
7360 				     unsigned int nrt6)
7361 {
7362 	struct mlxsw_sp_fib6_entry *fib6_entry;
7363 	struct mlxsw_sp_fib_node *fib_node;
7364 	struct fib6_info *rt = rt_arr[0];
7365 
7366 	if (mlxsw_sp_fib6_rt_should_ignore(rt))
7367 		return;
7368 
7369 	/* Multipath routes are first added to the FIB trie and only then
7370 	 * notified. If we vetoed the addition, we will get a delete
7371 	 * notification for a route we do not have. Therefore, do not warn if
7372 	 * route was not found.
7373 	 */
7374 	fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
7375 	if (!fib6_entry)
7376 		return;
7377 
7378 	/* If not all the nexthops are deleted, then only reduce the nexthop
7379 	 * group.
7380 	 */
7381 	if (nrt6 != fib6_entry->nrt6) {
7382 		mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt_arr,
7383 						nrt6);
7384 		return;
7385 	}
7386 
7387 	fib_node = fib6_entry->common.fib_node;
7388 
7389 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, &fib6_entry->common);
7390 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7391 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7392 }
7393 
7394 static struct mlxsw_sp_mr_table *
7395 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
7396 {
7397 	if (family == RTNL_FAMILY_IPMR)
7398 		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
7399 	else
7400 		return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
7401 }
7402 
7403 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
7404 				     struct mfc_entry_notifier_info *men_info,
7405 				     bool replace)
7406 {
7407 	struct mlxsw_sp_mr_table *mrt;
7408 	struct mlxsw_sp_vr *vr;
7409 
7410 	vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
7411 	if (IS_ERR(vr))
7412 		return PTR_ERR(vr);
7413 
7414 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7415 	return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
7416 }
7417 
7418 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
7419 				      struct mfc_entry_notifier_info *men_info)
7420 {
7421 	struct mlxsw_sp_mr_table *mrt;
7422 	struct mlxsw_sp_vr *vr;
7423 
7424 	vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
7425 	if (WARN_ON(!vr))
7426 		return;
7427 
7428 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
7429 	mlxsw_sp_mr_route_del(mrt, men_info->mfc);
7430 	mlxsw_sp_vr_put(mlxsw_sp, vr);
7431 }
7432 
7433 static int
7434 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
7435 			      struct vif_entry_notifier_info *ven_info)
7436 {
7437 	struct mlxsw_sp_mr_table *mrt;
7438 	struct mlxsw_sp_rif *rif;
7439 	struct mlxsw_sp_vr *vr;
7440 
7441 	vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
7442 	if (IS_ERR(vr))
7443 		return PTR_ERR(vr);
7444 
7445 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7446 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
7447 	return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
7448 				   ven_info->vif_index,
7449 				   ven_info->vif_flags, rif);
7450 }
7451 
7452 static void
7453 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
7454 			      struct vif_entry_notifier_info *ven_info)
7455 {
7456 	struct mlxsw_sp_mr_table *mrt;
7457 	struct mlxsw_sp_vr *vr;
7458 
7459 	vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
7460 	if (WARN_ON(!vr))
7461 		return;
7462 
7463 	mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
7464 	mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
7465 	mlxsw_sp_vr_put(mlxsw_sp, vr);
7466 }
7467 
7468 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
7469 				     struct mlxsw_sp_fib_node *fib_node)
7470 {
7471 	struct mlxsw_sp_fib4_entry *fib4_entry;
7472 
7473 	fib4_entry = container_of(fib_node->fib_entry,
7474 				  struct mlxsw_sp_fib4_entry, common);
7475 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7476 	mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
7477 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7478 }
7479 
7480 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
7481 				     struct mlxsw_sp_fib_node *fib_node)
7482 {
7483 	struct mlxsw_sp_fib6_entry *fib6_entry;
7484 
7485 	fib6_entry = container_of(fib_node->fib_entry,
7486 				  struct mlxsw_sp_fib6_entry, common);
7487 	mlxsw_sp_fib_node_entry_unlink(mlxsw_sp, fib_node->fib_entry);
7488 	mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
7489 	mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
7490 }
7491 
7492 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
7493 				    struct mlxsw_sp_fib_node *fib_node)
7494 {
7495 	switch (fib_node->fib->proto) {
7496 	case MLXSW_SP_L3_PROTO_IPV4:
7497 		mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
7498 		break;
7499 	case MLXSW_SP_L3_PROTO_IPV6:
7500 		mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
7501 		break;
7502 	}
7503 }
7504 
7505 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
7506 				  struct mlxsw_sp_vr *vr,
7507 				  enum mlxsw_sp_l3proto proto)
7508 {
7509 	struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
7510 	struct mlxsw_sp_fib_node *fib_node, *tmp;
7511 
7512 	list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
7513 		bool do_break = &tmp->list == &fib->node_list;
7514 
7515 		mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
7516 		if (do_break)
7517 			break;
7518 	}
7519 }
7520 
7521 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
7522 {
7523 	int max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
7524 	int i, j;
7525 
7526 	for (i = 0; i < max_vrs; i++) {
7527 		struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
7528 
7529 		if (!mlxsw_sp_vr_is_used(vr))
7530 			continue;
7531 
7532 		for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
7533 			mlxsw_sp_mr_table_flush(vr->mr_table[j]);
7534 		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
7535 
7536 		/* If virtual router was only used for IPv4, then it's no
7537 		 * longer used.
7538 		 */
7539 		if (!mlxsw_sp_vr_is_used(vr))
7540 			continue;
7541 		mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
7542 	}
7543 }
7544 
7545 struct mlxsw_sp_fib6_event_work {
7546 	struct fib6_info **rt_arr;
7547 	unsigned int nrt6;
7548 };
7549 
7550 struct mlxsw_sp_fib_event_work {
7551 	struct work_struct work;
7552 	netdevice_tracker dev_tracker;
7553 	union {
7554 		struct mlxsw_sp_fib6_event_work fib6_work;
7555 		struct fib_entry_notifier_info fen_info;
7556 		struct fib_rule_notifier_info fr_info;
7557 		struct fib_nh_notifier_info fnh_info;
7558 		struct mfc_entry_notifier_info men_info;
7559 		struct vif_entry_notifier_info ven_info;
7560 	};
7561 	struct mlxsw_sp *mlxsw_sp;
7562 	unsigned long event;
7563 };
7564 
7565 static int
7566 mlxsw_sp_router_fib6_work_init(struct mlxsw_sp_fib6_event_work *fib6_work,
7567 			       struct fib6_entry_notifier_info *fen6_info)
7568 {
7569 	struct fib6_info *rt = fen6_info->rt;
7570 	struct fib6_info **rt_arr;
7571 	struct fib6_info *iter;
7572 	unsigned int nrt6;
7573 	int i = 0;
7574 
7575 	nrt6 = fen6_info->nsiblings + 1;
7576 
7577 	rt_arr = kcalloc(nrt6, sizeof(struct fib6_info *), GFP_ATOMIC);
7578 	if (!rt_arr)
7579 		return -ENOMEM;
7580 
7581 	fib6_work->rt_arr = rt_arr;
7582 	fib6_work->nrt6 = nrt6;
7583 
7584 	rt_arr[0] = rt;
7585 	fib6_info_hold(rt);
7586 
7587 	if (!fen6_info->nsiblings)
7588 		return 0;
7589 
7590 	list_for_each_entry(iter, &rt->fib6_siblings, fib6_siblings) {
7591 		if (i == fen6_info->nsiblings)
7592 			break;
7593 
7594 		rt_arr[i + 1] = iter;
7595 		fib6_info_hold(iter);
7596 		i++;
7597 	}
7598 	WARN_ON_ONCE(i != fen6_info->nsiblings);
7599 
7600 	return 0;
7601 }
7602 
7603 static void
7604 mlxsw_sp_router_fib6_work_fini(struct mlxsw_sp_fib6_event_work *fib6_work)
7605 {
7606 	int i;
7607 
7608 	for (i = 0; i < fib6_work->nrt6; i++)
7609 		mlxsw_sp_rt6_release(fib6_work->rt_arr[i]);
7610 	kfree(fib6_work->rt_arr);
7611 }
7612 
7613 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
7614 {
7615 	struct mlxsw_sp_fib_event_work *fib_work =
7616 		container_of(work, struct mlxsw_sp_fib_event_work, work);
7617 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7618 	int err;
7619 
7620 	mutex_lock(&mlxsw_sp->router->lock);
7621 	mlxsw_sp_span_respin(mlxsw_sp);
7622 
7623 	switch (fib_work->event) {
7624 	case FIB_EVENT_ENTRY_REPLACE:
7625 		err = mlxsw_sp_router_fib4_replace(mlxsw_sp,
7626 						   &fib_work->fen_info);
7627 		if (err) {
7628 			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7629 			mlxsw_sp_fib4_offload_failed_flag_set(mlxsw_sp,
7630 							      &fib_work->fen_info);
7631 		}
7632 		fib_info_put(fib_work->fen_info.fi);
7633 		break;
7634 	case FIB_EVENT_ENTRY_DEL:
7635 		mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
7636 		fib_info_put(fib_work->fen_info.fi);
7637 		break;
7638 	case FIB_EVENT_NH_ADD:
7639 	case FIB_EVENT_NH_DEL:
7640 		mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
7641 					fib_work->fnh_info.fib_nh);
7642 		fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
7643 		break;
7644 	}
7645 	mutex_unlock(&mlxsw_sp->router->lock);
7646 	kfree(fib_work);
7647 }
7648 
7649 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
7650 {
7651 	struct mlxsw_sp_fib_event_work *fib_work =
7652 		    container_of(work, struct mlxsw_sp_fib_event_work, work);
7653 	struct mlxsw_sp_fib6_event_work *fib6_work = &fib_work->fib6_work;
7654 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7655 	int err;
7656 
7657 	mutex_lock(&mlxsw_sp->router->lock);
7658 	mlxsw_sp_span_respin(mlxsw_sp);
7659 
7660 	switch (fib_work->event) {
7661 	case FIB_EVENT_ENTRY_REPLACE:
7662 		err = mlxsw_sp_router_fib6_replace(mlxsw_sp,
7663 						   fib6_work->rt_arr,
7664 						   fib6_work->nrt6);
7665 		if (err) {
7666 			dev_warn(mlxsw_sp->bus_info->dev, "FIB replace failed.\n");
7667 			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7668 							      fib6_work->rt_arr,
7669 							      fib6_work->nrt6);
7670 		}
7671 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7672 		break;
7673 	case FIB_EVENT_ENTRY_APPEND:
7674 		err = mlxsw_sp_router_fib6_append(mlxsw_sp,
7675 						  fib6_work->rt_arr,
7676 						  fib6_work->nrt6);
7677 		if (err) {
7678 			dev_warn(mlxsw_sp->bus_info->dev, "FIB append failed.\n");
7679 			mlxsw_sp_fib6_offload_failed_flag_set(mlxsw_sp,
7680 							      fib6_work->rt_arr,
7681 							      fib6_work->nrt6);
7682 		}
7683 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7684 		break;
7685 	case FIB_EVENT_ENTRY_DEL:
7686 		mlxsw_sp_router_fib6_del(mlxsw_sp,
7687 					 fib6_work->rt_arr,
7688 					 fib6_work->nrt6);
7689 		mlxsw_sp_router_fib6_work_fini(fib6_work);
7690 		break;
7691 	}
7692 	mutex_unlock(&mlxsw_sp->router->lock);
7693 	kfree(fib_work);
7694 }
7695 
7696 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
7697 {
7698 	struct mlxsw_sp_fib_event_work *fib_work =
7699 		container_of(work, struct mlxsw_sp_fib_event_work, work);
7700 	struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
7701 	bool replace;
7702 	int err;
7703 
7704 	rtnl_lock();
7705 	mutex_lock(&mlxsw_sp->router->lock);
7706 	switch (fib_work->event) {
7707 	case FIB_EVENT_ENTRY_REPLACE:
7708 	case FIB_EVENT_ENTRY_ADD:
7709 		replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
7710 
7711 		err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info,
7712 						replace);
7713 		if (err)
7714 			dev_warn(mlxsw_sp->bus_info->dev, "MR entry add failed.\n");
7715 		mr_cache_put(fib_work->men_info.mfc);
7716 		break;
7717 	case FIB_EVENT_ENTRY_DEL:
7718 		mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info);
7719 		mr_cache_put(fib_work->men_info.mfc);
7720 		break;
7721 	case FIB_EVENT_VIF_ADD:
7722 		err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
7723 						    &fib_work->ven_info);
7724 		if (err)
7725 			dev_warn(mlxsw_sp->bus_info->dev, "MR VIF add failed.\n");
7726 		netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker);
7727 		break;
7728 	case FIB_EVENT_VIF_DEL:
7729 		mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
7730 					      &fib_work->ven_info);
7731 		netdev_put(fib_work->ven_info.dev, &fib_work->dev_tracker);
7732 		break;
7733 	}
7734 	mutex_unlock(&mlxsw_sp->router->lock);
7735 	rtnl_unlock();
7736 	kfree(fib_work);
7737 }
7738 
7739 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
7740 				       struct fib_notifier_info *info)
7741 {
7742 	struct fib_entry_notifier_info *fen_info;
7743 	struct fib_nh_notifier_info *fnh_info;
7744 
7745 	switch (fib_work->event) {
7746 	case FIB_EVENT_ENTRY_REPLACE:
7747 	case FIB_EVENT_ENTRY_DEL:
7748 		fen_info = container_of(info, struct fib_entry_notifier_info,
7749 					info);
7750 		fib_work->fen_info = *fen_info;
7751 		/* Take reference on fib_info to prevent it from being
7752 		 * freed while work is queued. Release it afterwards.
7753 		 */
7754 		fib_info_hold(fib_work->fen_info.fi);
7755 		break;
7756 	case FIB_EVENT_NH_ADD:
7757 	case FIB_EVENT_NH_DEL:
7758 		fnh_info = container_of(info, struct fib_nh_notifier_info,
7759 					info);
7760 		fib_work->fnh_info = *fnh_info;
7761 		fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
7762 		break;
7763 	}
7764 }
7765 
7766 static int mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
7767 				      struct fib_notifier_info *info)
7768 {
7769 	struct fib6_entry_notifier_info *fen6_info;
7770 	int err;
7771 
7772 	switch (fib_work->event) {
7773 	case FIB_EVENT_ENTRY_REPLACE:
7774 	case FIB_EVENT_ENTRY_APPEND:
7775 	case FIB_EVENT_ENTRY_DEL:
7776 		fen6_info = container_of(info, struct fib6_entry_notifier_info,
7777 					 info);
7778 		err = mlxsw_sp_router_fib6_work_init(&fib_work->fib6_work,
7779 						     fen6_info);
7780 		if (err)
7781 			return err;
7782 		break;
7783 	}
7784 
7785 	return 0;
7786 }
7787 
7788 static void
7789 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
7790 			    struct fib_notifier_info *info)
7791 {
7792 	switch (fib_work->event) {
7793 	case FIB_EVENT_ENTRY_REPLACE:
7794 	case FIB_EVENT_ENTRY_ADD:
7795 	case FIB_EVENT_ENTRY_DEL:
7796 		memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
7797 		mr_cache_hold(fib_work->men_info.mfc);
7798 		break;
7799 	case FIB_EVENT_VIF_ADD:
7800 	case FIB_EVENT_VIF_DEL:
7801 		memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
7802 		netdev_hold(fib_work->ven_info.dev, &fib_work->dev_tracker,
7803 			    GFP_ATOMIC);
7804 		break;
7805 	}
7806 }
7807 
7808 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
7809 					  struct fib_notifier_info *info,
7810 					  struct mlxsw_sp *mlxsw_sp)
7811 {
7812 	struct netlink_ext_ack *extack = info->extack;
7813 	struct fib_rule_notifier_info *fr_info;
7814 	struct fib_rule *rule;
7815 	int err = 0;
7816 
7817 	/* nothing to do at the moment */
7818 	if (event == FIB_EVENT_RULE_DEL)
7819 		return 0;
7820 
7821 	fr_info = container_of(info, struct fib_rule_notifier_info, info);
7822 	rule = fr_info->rule;
7823 
7824 	/* Rule only affects locally generated traffic */
7825 	if (rule->iifindex == mlxsw_sp_net(mlxsw_sp)->loopback_dev->ifindex)
7826 		return 0;
7827 
7828 	switch (info->family) {
7829 	case AF_INET:
7830 		if (!fib4_rule_default(rule) && !rule->l3mdev)
7831 			err = -EOPNOTSUPP;
7832 		break;
7833 	case AF_INET6:
7834 		if (!fib6_rule_default(rule) && !rule->l3mdev)
7835 			err = -EOPNOTSUPP;
7836 		break;
7837 	case RTNL_FAMILY_IPMR:
7838 		if (!ipmr_rule_default(rule) && !rule->l3mdev)
7839 			err = -EOPNOTSUPP;
7840 		break;
7841 	case RTNL_FAMILY_IP6MR:
7842 		if (!ip6mr_rule_default(rule) && !rule->l3mdev)
7843 			err = -EOPNOTSUPP;
7844 		break;
7845 	}
7846 
7847 	if (err < 0)
7848 		NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
7849 
7850 	return err;
7851 }
7852 
7853 /* Called with rcu_read_lock() */
7854 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
7855 				     unsigned long event, void *ptr)
7856 {
7857 	struct mlxsw_sp_fib_event_work *fib_work;
7858 	struct fib_notifier_info *info = ptr;
7859 	struct mlxsw_sp_router *router;
7860 	int err;
7861 
7862 	if ((info->family != AF_INET && info->family != AF_INET6 &&
7863 	     info->family != RTNL_FAMILY_IPMR &&
7864 	     info->family != RTNL_FAMILY_IP6MR))
7865 		return NOTIFY_DONE;
7866 
7867 	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7868 
7869 	switch (event) {
7870 	case FIB_EVENT_RULE_ADD:
7871 	case FIB_EVENT_RULE_DEL:
7872 		err = mlxsw_sp_router_fib_rule_event(event, info,
7873 						     router->mlxsw_sp);
7874 		return notifier_from_errno(err);
7875 	case FIB_EVENT_ENTRY_ADD:
7876 	case FIB_EVENT_ENTRY_REPLACE:
7877 	case FIB_EVENT_ENTRY_APPEND:
7878 		if (info->family == AF_INET) {
7879 			struct fib_entry_notifier_info *fen_info = ptr;
7880 
7881 			if (fen_info->fi->fib_nh_is_v6) {
7882 				NL_SET_ERR_MSG_MOD(info->extack, "IPv6 gateway with IPv4 route is not supported");
7883 				return notifier_from_errno(-EINVAL);
7884 			}
7885 		}
7886 		break;
7887 	}
7888 
7889 	fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
7890 	if (!fib_work)
7891 		return NOTIFY_BAD;
7892 
7893 	fib_work->mlxsw_sp = router->mlxsw_sp;
7894 	fib_work->event = event;
7895 
7896 	switch (info->family) {
7897 	case AF_INET:
7898 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
7899 		mlxsw_sp_router_fib4_event(fib_work, info);
7900 		break;
7901 	case AF_INET6:
7902 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
7903 		err = mlxsw_sp_router_fib6_event(fib_work, info);
7904 		if (err)
7905 			goto err_fib_event;
7906 		break;
7907 	case RTNL_FAMILY_IP6MR:
7908 	case RTNL_FAMILY_IPMR:
7909 		INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
7910 		mlxsw_sp_router_fibmr_event(fib_work, info);
7911 		break;
7912 	}
7913 
7914 	mlxsw_core_schedule_work(&fib_work->work);
7915 
7916 	return NOTIFY_DONE;
7917 
7918 err_fib_event:
7919 	kfree(fib_work);
7920 	return NOTIFY_BAD;
7921 }
7922 
7923 static struct mlxsw_sp_rif *
7924 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
7925 			 const struct net_device *dev)
7926 {
7927 	int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7928 	int i;
7929 
7930 	for (i = 0; i < max_rifs; i++)
7931 		if (mlxsw_sp->router->rifs[i] &&
7932 		    mlxsw_sp_rif_dev_is(mlxsw_sp->router->rifs[i], dev))
7933 			return mlxsw_sp->router->rifs[i];
7934 
7935 	return NULL;
7936 }
7937 
7938 bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp,
7939 			 const struct net_device *dev)
7940 {
7941 	struct mlxsw_sp_rif *rif;
7942 
7943 	mutex_lock(&mlxsw_sp->router->lock);
7944 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7945 	mutex_unlock(&mlxsw_sp->router->lock);
7946 
7947 	return rif;
7948 }
7949 
7950 u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev)
7951 {
7952 	struct mlxsw_sp_rif *rif;
7953 	u16 vid = 0;
7954 
7955 	mutex_lock(&mlxsw_sp->router->lock);
7956 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7957 	if (!rif)
7958 		goto out;
7959 
7960 	/* We only return the VID for VLAN RIFs. Otherwise we return an
7961 	 * invalid value (0).
7962 	 */
7963 	if (rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN)
7964 		goto out;
7965 
7966 	vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7967 
7968 out:
7969 	mutex_unlock(&mlxsw_sp->router->lock);
7970 	return vid;
7971 }
7972 
7973 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
7974 {
7975 	char ritr_pl[MLXSW_REG_RITR_LEN];
7976 	int err;
7977 
7978 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
7979 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7980 	if (err)
7981 		return err;
7982 
7983 	mlxsw_reg_ritr_enable_set(ritr_pl, false);
7984 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7985 }
7986 
7987 static int mlxsw_sp_router_rif_made_sync(struct mlxsw_sp *mlxsw_sp,
7988 					 struct mlxsw_sp_rif *rif)
7989 {
7990 	int err;
7991 
7992 	err = mlxsw_sp_neigh_rif_made_sync(mlxsw_sp, rif);
7993 	if (err)
7994 		return err;
7995 
7996 	err = mlxsw_sp_nexthop_rif_made_sync(mlxsw_sp, rif);
7997 	if (err)
7998 		goto err_nexthop;
7999 
8000 	return 0;
8001 
8002 err_nexthop:
8003 	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8004 	return err;
8005 }
8006 
8007 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
8008 					  struct mlxsw_sp_rif *rif)
8009 {
8010 	/* Signal to nexthop cleanup that the RIF is going away. */
8011 	rif->crif->rif = NULL;
8012 
8013 	mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
8014 	mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
8015 	mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
8016 }
8017 
8018 static bool __mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8019 {
8020 	struct inet6_dev *inet6_dev;
8021 	struct in_device *idev;
8022 
8023 	idev = __in_dev_get_rcu(dev);
8024 	if (idev && idev->ifa_list)
8025 		return false;
8026 
8027 	inet6_dev = __in6_dev_get(dev);
8028 	if (inet6_dev && !list_empty(&inet6_dev->addr_list))
8029 		return false;
8030 
8031 	return true;
8032 }
8033 
8034 static bool mlxsw_sp_dev_addr_list_empty(const struct net_device *dev)
8035 {
8036 	bool addr_list_empty;
8037 
8038 	rcu_read_lock();
8039 	addr_list_empty = __mlxsw_sp_dev_addr_list_empty(dev);
8040 	rcu_read_unlock();
8041 
8042 	return addr_list_empty;
8043 }
8044 
8045 static bool
8046 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
8047 			   unsigned long event)
8048 {
8049 	bool addr_list_empty;
8050 
8051 	switch (event) {
8052 	case NETDEV_UP:
8053 		return rif == NULL;
8054 	case NETDEV_DOWN:
8055 		addr_list_empty = mlxsw_sp_dev_addr_list_empty(dev);
8056 
8057 		/* macvlans do not have a RIF, but rather piggy back on the
8058 		 * RIF of their lower device.
8059 		 */
8060 		if (netif_is_macvlan(dev) && addr_list_empty)
8061 			return true;
8062 
8063 		if (rif && addr_list_empty &&
8064 		    !netif_is_l3_slave(mlxsw_sp_rif_dev(rif)))
8065 			return true;
8066 		/* It is possible we already removed the RIF ourselves
8067 		 * if it was assigned to a netdev that is now a bridge
8068 		 * or LAG slave.
8069 		 */
8070 		return false;
8071 	}
8072 
8073 	return false;
8074 }
8075 
8076 static enum mlxsw_sp_rif_type
8077 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
8078 		      const struct net_device *dev)
8079 {
8080 	enum mlxsw_sp_fid_type type;
8081 
8082 	if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
8083 		return MLXSW_SP_RIF_TYPE_IPIP_LB;
8084 
8085 	/* Otherwise RIF type is derived from the type of the underlying FID. */
8086 	if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
8087 		type = MLXSW_SP_FID_TYPE_8021Q;
8088 	else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
8089 		type = MLXSW_SP_FID_TYPE_8021Q;
8090 	else if (netif_is_bridge_master(dev))
8091 		type = MLXSW_SP_FID_TYPE_8021D;
8092 	else
8093 		type = MLXSW_SP_FID_TYPE_RFID;
8094 
8095 	return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
8096 }
8097 
8098 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index,
8099 				    u8 rif_entries)
8100 {
8101 	*p_rif_index = gen_pool_alloc(mlxsw_sp->router->rifs_table,
8102 				      rif_entries);
8103 	if (*p_rif_index == 0)
8104 		return -ENOBUFS;
8105 	*p_rif_index -= MLXSW_SP_ROUTER_GENALLOC_OFFSET;
8106 
8107 	/* RIF indexes must be aligned to the allocation size. */
8108 	WARN_ON_ONCE(*p_rif_index % rif_entries);
8109 
8110 	return 0;
8111 }
8112 
8113 static void mlxsw_sp_rif_index_free(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
8114 				    u8 rif_entries)
8115 {
8116 	gen_pool_free(mlxsw_sp->router->rifs_table,
8117 		      MLXSW_SP_ROUTER_GENALLOC_OFFSET + rif_index, rif_entries);
8118 }
8119 
8120 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
8121 					       u16 vr_id,
8122 					       struct mlxsw_sp_crif *crif)
8123 {
8124 	struct net_device *l3_dev = crif ? crif->key.dev : NULL;
8125 	struct mlxsw_sp_rif *rif;
8126 
8127 	rif = kzalloc(rif_size, GFP_KERNEL);
8128 	if (!rif)
8129 		return NULL;
8130 
8131 	INIT_LIST_HEAD(&rif->neigh_list);
8132 	if (l3_dev) {
8133 		ether_addr_copy(rif->addr, l3_dev->dev_addr);
8134 		rif->mtu = l3_dev->mtu;
8135 	}
8136 	rif->vr_id = vr_id;
8137 	rif->rif_index = rif_index;
8138 	if (crif) {
8139 		rif->crif = crif;
8140 		crif->rif = rif;
8141 	}
8142 
8143 	return rif;
8144 }
8145 
8146 static void mlxsw_sp_rif_free(struct mlxsw_sp_rif *rif)
8147 {
8148 	WARN_ON(!list_empty(&rif->neigh_list));
8149 
8150 	if (rif->crif)
8151 		rif->crif->rif = NULL;
8152 	kfree(rif);
8153 }
8154 
8155 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
8156 					   u16 rif_index)
8157 {
8158 	return mlxsw_sp->router->rifs[rif_index];
8159 }
8160 
8161 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
8162 {
8163 	return rif->rif_index;
8164 }
8165 
8166 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8167 {
8168 	return lb_rif->common.rif_index;
8169 }
8170 
8171 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8172 {
8173 	struct net_device *dev = mlxsw_sp_rif_dev(&lb_rif->common);
8174 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
8175 	struct mlxsw_sp_vr *ul_vr;
8176 
8177 	ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
8178 	if (WARN_ON(IS_ERR(ul_vr)))
8179 		return 0;
8180 
8181 	return ul_vr->id;
8182 }
8183 
8184 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
8185 {
8186 	return lb_rif->ul_rif_id;
8187 }
8188 
8189 static bool
8190 mlxsw_sp_router_port_l3_stats_enabled(struct mlxsw_sp_rif *rif)
8191 {
8192 	return mlxsw_sp_rif_counter_valid_get(rif,
8193 					      MLXSW_SP_RIF_COUNTER_EGRESS) &&
8194 	       mlxsw_sp_rif_counter_valid_get(rif,
8195 					      MLXSW_SP_RIF_COUNTER_INGRESS);
8196 }
8197 
8198 static int
8199 mlxsw_sp_router_port_l3_stats_enable(struct mlxsw_sp_rif *rif)
8200 {
8201 	int err;
8202 
8203 	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8204 	if (err)
8205 		return err;
8206 
8207 	/* Clear stale data. */
8208 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8209 					       MLXSW_SP_RIF_COUNTER_INGRESS,
8210 					       NULL);
8211 	if (err)
8212 		goto err_clear_ingress;
8213 
8214 	err = mlxsw_sp_rif_counter_alloc(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8215 	if (err)
8216 		goto err_alloc_egress;
8217 
8218 	/* Clear stale data. */
8219 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8220 					       MLXSW_SP_RIF_COUNTER_EGRESS,
8221 					       NULL);
8222 	if (err)
8223 		goto err_clear_egress;
8224 
8225 	return 0;
8226 
8227 err_clear_egress:
8228 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8229 err_alloc_egress:
8230 err_clear_ingress:
8231 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8232 	return err;
8233 }
8234 
8235 static void
8236 mlxsw_sp_router_port_l3_stats_disable(struct mlxsw_sp_rif *rif)
8237 {
8238 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_EGRESS);
8239 	mlxsw_sp_rif_counter_free(rif, MLXSW_SP_RIF_COUNTER_INGRESS);
8240 }
8241 
8242 static void
8243 mlxsw_sp_router_port_l3_stats_report_used(struct mlxsw_sp_rif *rif,
8244 					  struct netdev_notifier_offload_xstats_info *info)
8245 {
8246 	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8247 		return;
8248 	netdev_offload_xstats_report_used(info->report_used);
8249 }
8250 
8251 static int
8252 mlxsw_sp_router_port_l3_stats_fetch(struct mlxsw_sp_rif *rif,
8253 				    struct rtnl_hw_stats64 *p_stats)
8254 {
8255 	struct mlxsw_sp_rif_counter_set_basic ingress;
8256 	struct mlxsw_sp_rif_counter_set_basic egress;
8257 	int err;
8258 
8259 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8260 					       MLXSW_SP_RIF_COUNTER_INGRESS,
8261 					       &ingress);
8262 	if (err)
8263 		return err;
8264 
8265 	err = mlxsw_sp_rif_counter_fetch_clear(rif,
8266 					       MLXSW_SP_RIF_COUNTER_EGRESS,
8267 					       &egress);
8268 	if (err)
8269 		return err;
8270 
8271 #define MLXSW_SP_ROUTER_ALL_GOOD(SET, SFX)		\
8272 		((SET.good_unicast_ ## SFX) +		\
8273 		 (SET.good_multicast_ ## SFX) +		\
8274 		 (SET.good_broadcast_ ## SFX))
8275 
8276 	p_stats->rx_packets = MLXSW_SP_ROUTER_ALL_GOOD(ingress, packets);
8277 	p_stats->tx_packets = MLXSW_SP_ROUTER_ALL_GOOD(egress, packets);
8278 	p_stats->rx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(ingress, bytes);
8279 	p_stats->tx_bytes = MLXSW_SP_ROUTER_ALL_GOOD(egress, bytes);
8280 	p_stats->rx_errors = ingress.error_packets;
8281 	p_stats->tx_errors = egress.error_packets;
8282 	p_stats->rx_dropped = ingress.discard_packets;
8283 	p_stats->tx_dropped = egress.discard_packets;
8284 	p_stats->multicast = ingress.good_multicast_packets +
8285 			     ingress.good_broadcast_packets;
8286 
8287 #undef MLXSW_SP_ROUTER_ALL_GOOD
8288 
8289 	return 0;
8290 }
8291 
8292 static int
8293 mlxsw_sp_router_port_l3_stats_report_delta(struct mlxsw_sp_rif *rif,
8294 					   struct netdev_notifier_offload_xstats_info *info)
8295 {
8296 	struct rtnl_hw_stats64 stats = {};
8297 	int err;
8298 
8299 	if (!mlxsw_sp_router_port_l3_stats_enabled(rif))
8300 		return 0;
8301 
8302 	err = mlxsw_sp_router_port_l3_stats_fetch(rif, &stats);
8303 	if (err)
8304 		return err;
8305 
8306 	netdev_offload_xstats_report_delta(info->report_delta, &stats);
8307 	return 0;
8308 }
8309 
8310 struct mlxsw_sp_router_hwstats_notify_work {
8311 	struct work_struct work;
8312 	struct net_device *dev;
8313 	netdevice_tracker dev_tracker;
8314 };
8315 
8316 static void mlxsw_sp_router_hwstats_notify_work(struct work_struct *work)
8317 {
8318 	struct mlxsw_sp_router_hwstats_notify_work *hws_work =
8319 		container_of(work, struct mlxsw_sp_router_hwstats_notify_work,
8320 			     work);
8321 
8322 	rtnl_lock();
8323 	rtnl_offload_xstats_notify(hws_work->dev);
8324 	rtnl_unlock();
8325 	netdev_put(hws_work->dev, &hws_work->dev_tracker);
8326 	kfree(hws_work);
8327 }
8328 
8329 static void
8330 mlxsw_sp_router_hwstats_notify_schedule(struct net_device *dev)
8331 {
8332 	struct mlxsw_sp_router_hwstats_notify_work *hws_work;
8333 
8334 	/* To collect notification payload, the core ends up sending another
8335 	 * notifier block message, which would deadlock on the attempt to
8336 	 * acquire the router lock again. Just postpone the notification until
8337 	 * later.
8338 	 */
8339 
8340 	hws_work = kzalloc(sizeof(*hws_work), GFP_KERNEL);
8341 	if (!hws_work)
8342 		return;
8343 
8344 	INIT_WORK(&hws_work->work, mlxsw_sp_router_hwstats_notify_work);
8345 	netdev_hold(dev, &hws_work->dev_tracker, GFP_KERNEL);
8346 	hws_work->dev = dev;
8347 	mlxsw_core_schedule_work(&hws_work->work);
8348 }
8349 
8350 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
8351 {
8352 	return mlxsw_sp_rif_dev(rif)->ifindex;
8353 }
8354 
8355 bool mlxsw_sp_rif_has_dev(const struct mlxsw_sp_rif *rif)
8356 {
8357 	return !!mlxsw_sp_rif_dev(rif);
8358 }
8359 
8360 bool mlxsw_sp_rif_dev_is(const struct mlxsw_sp_rif *rif,
8361 			 const struct net_device *dev)
8362 {
8363 	return mlxsw_sp_rif_dev(rif) == dev;
8364 }
8365 
8366 static void mlxsw_sp_rif_push_l3_stats(struct mlxsw_sp_rif *rif)
8367 {
8368 	struct rtnl_hw_stats64 stats = {};
8369 
8370 	if (!mlxsw_sp_router_port_l3_stats_fetch(rif, &stats))
8371 		netdev_offload_xstats_push_delta(mlxsw_sp_rif_dev(rif),
8372 						 NETDEV_OFFLOAD_XSTATS_TYPE_L3,
8373 						 &stats);
8374 }
8375 
8376 static struct mlxsw_sp_rif *
8377 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
8378 		    const struct mlxsw_sp_rif_params *params,
8379 		    struct netlink_ext_ack *extack)
8380 {
8381 	u8 rif_entries = params->double_entry ? 2 : 1;
8382 	u32 tb_id = l3mdev_fib_table(params->dev);
8383 	const struct mlxsw_sp_rif_ops *ops;
8384 	struct mlxsw_sp_fid *fid = NULL;
8385 	enum mlxsw_sp_rif_type type;
8386 	struct mlxsw_sp_crif *crif;
8387 	struct mlxsw_sp_rif *rif;
8388 	struct mlxsw_sp_vr *vr;
8389 	u16 rif_index;
8390 	int i, err;
8391 
8392 	type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
8393 	ops = mlxsw_sp->router->rif_ops_arr[type];
8394 
8395 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
8396 	if (IS_ERR(vr))
8397 		return ERR_CAST(vr);
8398 	vr->rif_count++;
8399 
8400 	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
8401 	if (err) {
8402 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
8403 		goto err_rif_index_alloc;
8404 	}
8405 
8406 	crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, params->dev);
8407 	if (WARN_ON(!crif)) {
8408 		err = -ENOENT;
8409 		goto err_crif_lookup;
8410 	}
8411 
8412 	rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, crif);
8413 	if (!rif) {
8414 		err = -ENOMEM;
8415 		goto err_rif_alloc;
8416 	}
8417 	netdev_hold(params->dev, &rif->dev_tracker, GFP_KERNEL);
8418 	mlxsw_sp->router->rifs[rif_index] = rif;
8419 	rif->mlxsw_sp = mlxsw_sp;
8420 	rif->ops = ops;
8421 	rif->rif_entries = rif_entries;
8422 
8423 	if (ops->fid_get) {
8424 		fid = ops->fid_get(rif, params, extack);
8425 		if (IS_ERR(fid)) {
8426 			err = PTR_ERR(fid);
8427 			goto err_fid_get;
8428 		}
8429 		rif->fid = fid;
8430 	}
8431 
8432 	if (ops->setup)
8433 		ops->setup(rif, params);
8434 
8435 	err = ops->configure(rif, extack);
8436 	if (err)
8437 		goto err_configure;
8438 
8439 	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
8440 		err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
8441 		if (err)
8442 			goto err_mr_rif_add;
8443 	}
8444 
8445 	err = mlxsw_sp_router_rif_made_sync(mlxsw_sp, rif);
8446 	if (err)
8447 		goto err_rif_made_sync;
8448 
8449 	if (netdev_offload_xstats_enabled(params->dev,
8450 					  NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8451 		err = mlxsw_sp_router_port_l3_stats_enable(rif);
8452 		if (err)
8453 			goto err_stats_enable;
8454 		mlxsw_sp_router_hwstats_notify_schedule(params->dev);
8455 	} else {
8456 		mlxsw_sp_rif_counters_alloc(rif);
8457 	}
8458 
8459 	atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
8460 	return rif;
8461 
8462 err_stats_enable:
8463 	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8464 err_rif_made_sync:
8465 err_mr_rif_add:
8466 	for (i--; i >= 0; i--)
8467 		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8468 	ops->deconfigure(rif);
8469 err_configure:
8470 	if (fid)
8471 		mlxsw_sp_fid_put(fid);
8472 err_fid_get:
8473 	mlxsw_sp->router->rifs[rif_index] = NULL;
8474 	netdev_put(params->dev, &rif->dev_tracker);
8475 	mlxsw_sp_rif_free(rif);
8476 err_rif_alloc:
8477 err_crif_lookup:
8478 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8479 err_rif_index_alloc:
8480 	vr->rif_count--;
8481 	mlxsw_sp_vr_put(mlxsw_sp, vr);
8482 	return ERR_PTR(err);
8483 }
8484 
8485 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
8486 {
8487 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
8488 	const struct mlxsw_sp_rif_ops *ops = rif->ops;
8489 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8490 	struct mlxsw_sp_crif *crif = rif->crif;
8491 	struct mlxsw_sp_fid *fid = rif->fid;
8492 	u8 rif_entries = rif->rif_entries;
8493 	u16 rif_index = rif->rif_index;
8494 	struct mlxsw_sp_vr *vr;
8495 	int i;
8496 
8497 	atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
8498 	mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
8499 	vr = &mlxsw_sp->router->vrs[rif->vr_id];
8500 
8501 	if (netdev_offload_xstats_enabled(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3)) {
8502 		mlxsw_sp_rif_push_l3_stats(rif);
8503 		mlxsw_sp_router_port_l3_stats_disable(rif);
8504 		mlxsw_sp_router_hwstats_notify_schedule(dev);
8505 	} else {
8506 		mlxsw_sp_rif_counters_free(rif);
8507 	}
8508 
8509 	for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
8510 		mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
8511 	ops->deconfigure(rif);
8512 	if (fid)
8513 		/* Loopback RIFs are not associated with a FID. */
8514 		mlxsw_sp_fid_put(fid);
8515 	mlxsw_sp->router->rifs[rif->rif_index] = NULL;
8516 	netdev_put(dev, &rif->dev_tracker);
8517 	mlxsw_sp_rif_free(rif);
8518 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
8519 	vr->rif_count--;
8520 	mlxsw_sp_vr_put(mlxsw_sp, vr);
8521 
8522 	if (crif->can_destroy)
8523 		mlxsw_sp_crif_free(crif);
8524 }
8525 
8526 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
8527 				 struct net_device *dev)
8528 {
8529 	struct mlxsw_sp_rif *rif;
8530 
8531 	mutex_lock(&mlxsw_sp->router->lock);
8532 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
8533 	if (!rif)
8534 		goto out;
8535 	mlxsw_sp_rif_destroy(rif);
8536 out:
8537 	mutex_unlock(&mlxsw_sp->router->lock);
8538 }
8539 
8540 static void mlxsw_sp_rif_destroy_vlan_upper(struct mlxsw_sp *mlxsw_sp,
8541 					    struct net_device *br_dev,
8542 					    u16 vid)
8543 {
8544 	struct net_device *upper_dev;
8545 	struct mlxsw_sp_crif *crif;
8546 
8547 	rcu_read_lock();
8548 	upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q), vid);
8549 	rcu_read_unlock();
8550 
8551 	if (!upper_dev)
8552 		return;
8553 
8554 	crif = mlxsw_sp_crif_lookup(mlxsw_sp->router, upper_dev);
8555 	if (!crif || !crif->rif)
8556 		return;
8557 
8558 	mlxsw_sp_rif_destroy(crif->rif);
8559 }
8560 
8561 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
8562 					  struct net_device *l3_dev,
8563 					  int lower_pvid,
8564 					  unsigned long event,
8565 					  struct netlink_ext_ack *extack);
8566 
8567 int mlxsw_sp_router_bridge_vlan_add(struct mlxsw_sp *mlxsw_sp,
8568 				    struct net_device *br_dev,
8569 				    u16 new_vid, bool is_pvid,
8570 				    struct netlink_ext_ack *extack)
8571 {
8572 	struct mlxsw_sp_rif *old_rif;
8573 	struct mlxsw_sp_rif *new_rif;
8574 	struct net_device *upper_dev;
8575 	u16 old_pvid = 0;
8576 	u16 new_pvid;
8577 	int err = 0;
8578 
8579 	mutex_lock(&mlxsw_sp->router->lock);
8580 	old_rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, br_dev);
8581 	if (old_rif) {
8582 		/* If the RIF on the bridge is not a VLAN RIF, we shouldn't have
8583 		 * gotten a PVID notification.
8584 		 */
8585 		if (WARN_ON(old_rif->ops->type != MLXSW_SP_RIF_TYPE_VLAN))
8586 			old_rif = NULL;
8587 		else
8588 			old_pvid = mlxsw_sp_fid_8021q_vid(old_rif->fid);
8589 	}
8590 
8591 	if (is_pvid)
8592 		new_pvid = new_vid;
8593 	else if (old_pvid == new_vid)
8594 		new_pvid = 0;
8595 	else
8596 		goto out;
8597 
8598 	if (old_pvid == new_pvid)
8599 		goto out;
8600 
8601 	if (new_pvid) {
8602 		struct mlxsw_sp_rif_params params = {
8603 			.dev = br_dev,
8604 			.vid = new_pvid,
8605 		};
8606 
8607 		/* If there is a VLAN upper with the same VID as the new PVID,
8608 		 * kill its RIF, if there is one.
8609 		 */
8610 		mlxsw_sp_rif_destroy_vlan_upper(mlxsw_sp, br_dev, new_pvid);
8611 
8612 		if (mlxsw_sp_dev_addr_list_empty(br_dev))
8613 			goto out;
8614 		new_rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
8615 		if (IS_ERR(new_rif)) {
8616 			err = PTR_ERR(new_rif);
8617 			goto out;
8618 		}
8619 
8620 		if (old_pvid)
8621 			mlxsw_sp_rif_migrate_destroy(mlxsw_sp, old_rif, new_rif,
8622 						     true);
8623 	} else {
8624 		mlxsw_sp_rif_destroy(old_rif);
8625 	}
8626 
8627 	if (old_pvid) {
8628 		rcu_read_lock();
8629 		upper_dev = __vlan_find_dev_deep_rcu(br_dev, htons(ETH_P_8021Q),
8630 						     old_pvid);
8631 		rcu_read_unlock();
8632 		if (upper_dev)
8633 			err = mlxsw_sp_inetaddr_bridge_event(mlxsw_sp,
8634 							     upper_dev,
8635 							     new_pvid,
8636 							     NETDEV_UP, extack);
8637 	}
8638 
8639 out:
8640 	mutex_unlock(&mlxsw_sp->router->lock);
8641 	return err;
8642 }
8643 
8644 static void
8645 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
8646 				 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8647 {
8648 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8649 
8650 	params->vid = mlxsw_sp_port_vlan->vid;
8651 	params->lag = mlxsw_sp_port->lagged;
8652 	if (params->lag)
8653 		params->lag_id = mlxsw_sp_port->lag_id;
8654 	else
8655 		params->system_port = mlxsw_sp_port->local_port;
8656 }
8657 
8658 static struct mlxsw_sp_rif_subport *
8659 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
8660 {
8661 	return container_of(rif, struct mlxsw_sp_rif_subport, common);
8662 }
8663 
8664 static struct mlxsw_sp_rif *
8665 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
8666 			 const struct mlxsw_sp_rif_params *params,
8667 			 struct netlink_ext_ack *extack)
8668 {
8669 	struct mlxsw_sp_rif_subport *rif_subport;
8670 	struct mlxsw_sp_rif *rif;
8671 
8672 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
8673 	if (!rif)
8674 		return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
8675 
8676 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8677 	refcount_inc(&rif_subport->ref_count);
8678 	return rif;
8679 }
8680 
8681 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
8682 {
8683 	struct mlxsw_sp_rif_subport *rif_subport;
8684 
8685 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
8686 	if (!refcount_dec_and_test(&rif_subport->ref_count))
8687 		return;
8688 
8689 	mlxsw_sp_rif_destroy(rif);
8690 }
8691 
8692 static int mlxsw_sp_rif_mac_profile_index_alloc(struct mlxsw_sp *mlxsw_sp,
8693 						struct mlxsw_sp_rif_mac_profile *profile,
8694 						struct netlink_ext_ack *extack)
8695 {
8696 	u8 max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
8697 	struct mlxsw_sp_router *router = mlxsw_sp->router;
8698 	int id;
8699 
8700 	id = idr_alloc(&router->rif_mac_profiles_idr, profile, 0,
8701 		       max_rif_mac_profiles, GFP_KERNEL);
8702 
8703 	if (id >= 0) {
8704 		profile->id = id;
8705 		return 0;
8706 	}
8707 
8708 	if (id == -ENOSPC)
8709 		NL_SET_ERR_MSG_MOD(extack,
8710 				   "Exceeded number of supported router interface MAC profiles");
8711 
8712 	return id;
8713 }
8714 
8715 static struct mlxsw_sp_rif_mac_profile *
8716 mlxsw_sp_rif_mac_profile_index_free(struct mlxsw_sp *mlxsw_sp, u8 mac_profile)
8717 {
8718 	struct mlxsw_sp_rif_mac_profile *profile;
8719 
8720 	profile = idr_remove(&mlxsw_sp->router->rif_mac_profiles_idr,
8721 			     mac_profile);
8722 	WARN_ON(!profile);
8723 	return profile;
8724 }
8725 
8726 static struct mlxsw_sp_rif_mac_profile *
8727 mlxsw_sp_rif_mac_profile_alloc(const char *mac)
8728 {
8729 	struct mlxsw_sp_rif_mac_profile *profile;
8730 
8731 	profile = kzalloc(sizeof(*profile), GFP_KERNEL);
8732 	if (!profile)
8733 		return NULL;
8734 
8735 	ether_addr_copy(profile->mac_prefix, mac);
8736 	refcount_set(&profile->ref_count, 1);
8737 	return profile;
8738 }
8739 
8740 static struct mlxsw_sp_rif_mac_profile *
8741 mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac)
8742 {
8743 	struct mlxsw_sp_router *router = mlxsw_sp->router;
8744 	struct mlxsw_sp_rif_mac_profile *profile;
8745 	int id;
8746 
8747 	idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) {
8748 		if (ether_addr_equal_masked(profile->mac_prefix, mac,
8749 					    mlxsw_sp->mac_mask))
8750 			return profile;
8751 	}
8752 
8753 	return NULL;
8754 }
8755 
8756 static u64 mlxsw_sp_rif_mac_profiles_occ_get(void *priv)
8757 {
8758 	const struct mlxsw_sp *mlxsw_sp = priv;
8759 
8760 	return atomic_read(&mlxsw_sp->router->rif_mac_profiles_count);
8761 }
8762 
8763 static u64 mlxsw_sp_rifs_occ_get(void *priv)
8764 {
8765 	const struct mlxsw_sp *mlxsw_sp = priv;
8766 
8767 	return atomic_read(&mlxsw_sp->router->rifs_count);
8768 }
8769 
8770 static struct mlxsw_sp_rif_mac_profile *
8771 mlxsw_sp_rif_mac_profile_create(struct mlxsw_sp *mlxsw_sp, const char *mac,
8772 				struct netlink_ext_ack *extack)
8773 {
8774 	struct mlxsw_sp_rif_mac_profile *profile;
8775 	int err;
8776 
8777 	profile = mlxsw_sp_rif_mac_profile_alloc(mac);
8778 	if (!profile)
8779 		return ERR_PTR(-ENOMEM);
8780 
8781 	err = mlxsw_sp_rif_mac_profile_index_alloc(mlxsw_sp, profile, extack);
8782 	if (err)
8783 		goto profile_index_alloc_err;
8784 
8785 	atomic_inc(&mlxsw_sp->router->rif_mac_profiles_count);
8786 	return profile;
8787 
8788 profile_index_alloc_err:
8789 	kfree(profile);
8790 	return ERR_PTR(err);
8791 }
8792 
8793 static void mlxsw_sp_rif_mac_profile_destroy(struct mlxsw_sp *mlxsw_sp,
8794 					     u8 mac_profile)
8795 {
8796 	struct mlxsw_sp_rif_mac_profile *profile;
8797 
8798 	atomic_dec(&mlxsw_sp->router->rif_mac_profiles_count);
8799 	profile = mlxsw_sp_rif_mac_profile_index_free(mlxsw_sp, mac_profile);
8800 	kfree(profile);
8801 }
8802 
8803 static int mlxsw_sp_rif_mac_profile_get(struct mlxsw_sp *mlxsw_sp,
8804 					const char *mac, u8 *p_mac_profile,
8805 					struct netlink_ext_ack *extack)
8806 {
8807 	struct mlxsw_sp_rif_mac_profile *profile;
8808 
8809 	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, mac);
8810 	if (profile) {
8811 		refcount_inc(&profile->ref_count);
8812 		goto out;
8813 	}
8814 
8815 	profile = mlxsw_sp_rif_mac_profile_create(mlxsw_sp, mac, extack);
8816 	if (IS_ERR(profile))
8817 		return PTR_ERR(profile);
8818 
8819 out:
8820 	*p_mac_profile = profile->id;
8821 	return 0;
8822 }
8823 
8824 static void mlxsw_sp_rif_mac_profile_put(struct mlxsw_sp *mlxsw_sp,
8825 					 u8 mac_profile)
8826 {
8827 	struct mlxsw_sp_rif_mac_profile *profile;
8828 
8829 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8830 			   mac_profile);
8831 	if (WARN_ON(!profile))
8832 		return;
8833 
8834 	if (!refcount_dec_and_test(&profile->ref_count))
8835 		return;
8836 
8837 	mlxsw_sp_rif_mac_profile_destroy(mlxsw_sp, mac_profile);
8838 }
8839 
8840 static bool mlxsw_sp_rif_mac_profile_is_shared(const struct mlxsw_sp_rif *rif)
8841 {
8842 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8843 	struct mlxsw_sp_rif_mac_profile *profile;
8844 
8845 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8846 			   rif->mac_profile_id);
8847 	if (WARN_ON(!profile))
8848 		return false;
8849 
8850 	return refcount_read(&profile->ref_count) > 1;
8851 }
8852 
8853 static int mlxsw_sp_rif_mac_profile_edit(struct mlxsw_sp_rif *rif,
8854 					 const char *new_mac)
8855 {
8856 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
8857 	struct mlxsw_sp_rif_mac_profile *profile;
8858 
8859 	profile = idr_find(&mlxsw_sp->router->rif_mac_profiles_idr,
8860 			   rif->mac_profile_id);
8861 	if (WARN_ON(!profile))
8862 		return -EINVAL;
8863 
8864 	ether_addr_copy(profile->mac_prefix, new_mac);
8865 	return 0;
8866 }
8867 
8868 static int
8869 mlxsw_sp_rif_mac_profile_replace(struct mlxsw_sp *mlxsw_sp,
8870 				 struct mlxsw_sp_rif *rif,
8871 				 const char *new_mac,
8872 				 struct netlink_ext_ack *extack)
8873 {
8874 	u8 mac_profile;
8875 	int err;
8876 
8877 	if (!mlxsw_sp_rif_mac_profile_is_shared(rif) &&
8878 	    !mlxsw_sp_rif_mac_profile_find(mlxsw_sp, new_mac))
8879 		return mlxsw_sp_rif_mac_profile_edit(rif, new_mac);
8880 
8881 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, new_mac,
8882 					   &mac_profile, extack);
8883 	if (err)
8884 		return err;
8885 
8886 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, rif->mac_profile_id);
8887 	rif->mac_profile_id = mac_profile;
8888 	return 0;
8889 }
8890 
8891 static int
8892 __mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8893 				 struct net_device *l3_dev,
8894 				 struct netlink_ext_ack *extack)
8895 {
8896 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8897 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
8898 	struct mlxsw_sp_rif_params params;
8899 	u16 vid = mlxsw_sp_port_vlan->vid;
8900 	struct mlxsw_sp_rif *rif;
8901 	struct mlxsw_sp_fid *fid;
8902 	int err;
8903 
8904 	params = (struct mlxsw_sp_rif_params) {
8905 		.dev = l3_dev,
8906 		.vid = vid,
8907 	};
8908 
8909 	mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
8910 	rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
8911 	if (IS_ERR(rif))
8912 		return PTR_ERR(rif);
8913 
8914 	/* FID was already created, just take a reference */
8915 	fid = rif->ops->fid_get(rif, &params, extack);
8916 	err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
8917 	if (err)
8918 		goto err_fid_port_vid_map;
8919 
8920 	err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
8921 	if (err)
8922 		goto err_port_vid_learning_set;
8923 
8924 	err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
8925 					BR_STATE_FORWARDING);
8926 	if (err)
8927 		goto err_port_vid_stp_set;
8928 
8929 	mlxsw_sp_port_vlan->fid = fid;
8930 
8931 	return 0;
8932 
8933 err_port_vid_stp_set:
8934 	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8935 err_port_vid_learning_set:
8936 	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8937 err_fid_port_vid_map:
8938 	mlxsw_sp_fid_put(fid);
8939 	mlxsw_sp_rif_subport_put(rif);
8940 	return err;
8941 }
8942 
8943 static void
8944 __mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8945 {
8946 	struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
8947 	struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
8948 	struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
8949 	u16 vid = mlxsw_sp_port_vlan->vid;
8950 
8951 	if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
8952 		return;
8953 
8954 	mlxsw_sp_port_vlan->fid = NULL;
8955 	mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
8956 	mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
8957 	mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
8958 	mlxsw_sp_fid_put(fid);
8959 	mlxsw_sp_rif_subport_put(rif);
8960 }
8961 
8962 static int
8963 mlxsw_sp_port_vlan_router_join_existing(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
8964 					struct net_device *l3_dev,
8965 					struct netlink_ext_ack *extack)
8966 {
8967 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8968 
8969 	lockdep_assert_held(&mlxsw_sp->router->lock);
8970 
8971 	if (!mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev))
8972 		return 0;
8973 
8974 	return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan, l3_dev,
8975 						extack);
8976 }
8977 
8978 void
8979 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
8980 {
8981 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port_vlan->mlxsw_sp_port->mlxsw_sp;
8982 
8983 	mutex_lock(&mlxsw_sp->router->lock);
8984 	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
8985 	mutex_unlock(&mlxsw_sp->router->lock);
8986 }
8987 
8988 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
8989 					     struct net_device *port_dev,
8990 					     unsigned long event, u16 vid,
8991 					     struct netlink_ext_ack *extack)
8992 {
8993 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
8994 	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
8995 
8996 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
8997 	if (WARN_ON(!mlxsw_sp_port_vlan))
8998 		return -EINVAL;
8999 
9000 	switch (event) {
9001 	case NETDEV_UP:
9002 		return __mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
9003 							l3_dev, extack);
9004 	case NETDEV_DOWN:
9005 		__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9006 		break;
9007 	}
9008 
9009 	return 0;
9010 }
9011 
9012 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
9013 					unsigned long event, bool nomaster,
9014 					struct netlink_ext_ack *extack)
9015 {
9016 	if (!nomaster && (netif_is_any_bridge_port(port_dev) ||
9017 			  netif_is_lag_port(port_dev)))
9018 		return 0;
9019 
9020 	return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
9021 						 MLXSW_SP_DEFAULT_VID, extack);
9022 }
9023 
9024 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
9025 					 struct net_device *lag_dev,
9026 					 unsigned long event, u16 vid,
9027 					 struct netlink_ext_ack *extack)
9028 {
9029 	struct net_device *port_dev;
9030 	struct list_head *iter;
9031 	int err;
9032 
9033 	netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
9034 		if (mlxsw_sp_port_dev_check(port_dev)) {
9035 			err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
9036 								port_dev,
9037 								event, vid,
9038 								extack);
9039 			if (err)
9040 				return err;
9041 		}
9042 	}
9043 
9044 	return 0;
9045 }
9046 
9047 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
9048 				       unsigned long event, bool nomaster,
9049 				       struct netlink_ext_ack *extack)
9050 {
9051 	if (!nomaster && netif_is_bridge_port(lag_dev))
9052 		return 0;
9053 
9054 	return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
9055 					     MLXSW_SP_DEFAULT_VID, extack);
9056 }
9057 
9058 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
9059 					  struct net_device *l3_dev,
9060 					  int lower_pvid,
9061 					  unsigned long event,
9062 					  struct netlink_ext_ack *extack)
9063 {
9064 	struct mlxsw_sp_rif_params params = {
9065 		.dev = l3_dev,
9066 	};
9067 	struct mlxsw_sp_rif *rif;
9068 	int err;
9069 
9070 	switch (event) {
9071 	case NETDEV_UP:
9072 		if (netif_is_bridge_master(l3_dev) && br_vlan_enabled(l3_dev)) {
9073 			u16 proto;
9074 
9075 			br_vlan_get_proto(l3_dev, &proto);
9076 			if (proto == ETH_P_8021AD) {
9077 				NL_SET_ERR_MSG_MOD(extack, "Adding an IP address to 802.1ad bridge is not supported");
9078 				return -EOPNOTSUPP;
9079 			}
9080 			err = br_vlan_get_pvid(l3_dev, &params.vid);
9081 			if (err)
9082 				return err;
9083 			if (!params.vid)
9084 				return 0;
9085 		} else if (is_vlan_dev(l3_dev)) {
9086 			params.vid = vlan_dev_vlan_id(l3_dev);
9087 
9088 			/* If the VID matches PVID of the bridge below, the
9089 			 * bridge owns the RIF for this VLAN. Don't do anything.
9090 			 */
9091 			if ((int)params.vid == lower_pvid)
9092 				return 0;
9093 		}
9094 
9095 		rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
9096 		if (IS_ERR(rif))
9097 			return PTR_ERR(rif);
9098 		break;
9099 	case NETDEV_DOWN:
9100 		rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9101 		mlxsw_sp_rif_destroy(rif);
9102 		break;
9103 	}
9104 
9105 	return 0;
9106 }
9107 
9108 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
9109 					struct net_device *vlan_dev,
9110 					unsigned long event, bool nomaster,
9111 					struct netlink_ext_ack *extack)
9112 {
9113 	struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
9114 	u16 vid = vlan_dev_vlan_id(vlan_dev);
9115 	u16 lower_pvid;
9116 	int err;
9117 
9118 	if (!nomaster && netif_is_bridge_port(vlan_dev))
9119 		return 0;
9120 
9121 	if (mlxsw_sp_port_dev_check(real_dev)) {
9122 		return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
9123 							 event, vid, extack);
9124 	} else if (netif_is_lag_master(real_dev)) {
9125 		return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
9126 						     vid, extack);
9127 	} else if (netif_is_bridge_master(real_dev) &&
9128 		   br_vlan_enabled(real_dev)) {
9129 		err = br_vlan_get_pvid(real_dev, &lower_pvid);
9130 		if (err)
9131 			return err;
9132 		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev,
9133 						      lower_pvid, event,
9134 						      extack);
9135 	}
9136 
9137 	return 0;
9138 }
9139 
9140 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
9141 {
9142 	u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
9143 	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9144 
9145 	return ether_addr_equal_masked(mac, vrrp4, mask);
9146 }
9147 
9148 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
9149 {
9150 	u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
9151 	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
9152 
9153 	return ether_addr_equal_masked(mac, vrrp6, mask);
9154 }
9155 
9156 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9157 				const u8 *mac, bool adding)
9158 {
9159 	char ritr_pl[MLXSW_REG_RITR_LEN];
9160 	u8 vrrp_id = adding ? mac[5] : 0;
9161 	int err;
9162 
9163 	if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
9164 	    !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
9165 		return 0;
9166 
9167 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
9168 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9169 	if (err)
9170 		return err;
9171 
9172 	if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
9173 		mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
9174 	else
9175 		mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
9176 
9177 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9178 }
9179 
9180 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
9181 				    const struct net_device *macvlan_dev,
9182 				    struct netlink_ext_ack *extack)
9183 {
9184 	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
9185 	struct mlxsw_sp_rif *rif;
9186 	int err;
9187 
9188 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
9189 	if (!rif)
9190 		return 0;
9191 
9192 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9193 				  mlxsw_sp_fid_index(rif->fid), true);
9194 	if (err)
9195 		return err;
9196 
9197 	err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
9198 				   macvlan_dev->dev_addr, true);
9199 	if (err)
9200 		goto err_rif_vrrp_add;
9201 
9202 	/* Make sure the bridge driver does not have this MAC pointing at
9203 	 * some other port.
9204 	 */
9205 	if (rif->ops->fdb_del)
9206 		rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
9207 
9208 	return 0;
9209 
9210 err_rif_vrrp_add:
9211 	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9212 			    mlxsw_sp_fid_index(rif->fid), false);
9213 	return err;
9214 }
9215 
9216 static void __mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9217 				       const struct net_device *macvlan_dev)
9218 {
9219 	struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
9220 	struct mlxsw_sp_rif *rif;
9221 
9222 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
9223 	/* If we do not have a RIF, then we already took care of
9224 	 * removing the macvlan's MAC during RIF deletion.
9225 	 */
9226 	if (!rif)
9227 		return;
9228 	mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
9229 			     false);
9230 	mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
9231 			    mlxsw_sp_fid_index(rif->fid), false);
9232 }
9233 
9234 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
9235 			      const struct net_device *macvlan_dev)
9236 {
9237 	mutex_lock(&mlxsw_sp->router->lock);
9238 	__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9239 	mutex_unlock(&mlxsw_sp->router->lock);
9240 }
9241 
9242 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
9243 					   struct net_device *macvlan_dev,
9244 					   unsigned long event,
9245 					   struct netlink_ext_ack *extack)
9246 {
9247 	switch (event) {
9248 	case NETDEV_UP:
9249 		return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
9250 	case NETDEV_DOWN:
9251 		__mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
9252 		break;
9253 	}
9254 
9255 	return 0;
9256 }
9257 
9258 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
9259 				     struct net_device *dev,
9260 				     unsigned long event, bool nomaster,
9261 				     struct netlink_ext_ack *extack)
9262 {
9263 	if (mlxsw_sp_port_dev_check(dev))
9264 		return mlxsw_sp_inetaddr_port_event(dev, event, nomaster,
9265 						    extack);
9266 	else if (netif_is_lag_master(dev))
9267 		return mlxsw_sp_inetaddr_lag_event(dev, event, nomaster,
9268 						   extack);
9269 	else if (netif_is_bridge_master(dev))
9270 		return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, -1, event,
9271 						      extack);
9272 	else if (is_vlan_dev(dev))
9273 		return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
9274 						    nomaster, extack);
9275 	else if (netif_is_macvlan(dev))
9276 		return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
9277 						       extack);
9278 	else
9279 		return 0;
9280 }
9281 
9282 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
9283 				   unsigned long event, void *ptr)
9284 {
9285 	struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
9286 	struct net_device *dev = ifa->ifa_dev->dev;
9287 	struct mlxsw_sp_router *router;
9288 	struct mlxsw_sp_rif *rif;
9289 	int err = 0;
9290 
9291 	/* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
9292 	if (event == NETDEV_UP)
9293 		return NOTIFY_DONE;
9294 
9295 	router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
9296 	mutex_lock(&router->lock);
9297 	rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
9298 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9299 		goto out;
9300 
9301 	err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, false,
9302 					NULL);
9303 out:
9304 	mutex_unlock(&router->lock);
9305 	return notifier_from_errno(err);
9306 }
9307 
9308 static int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
9309 					 unsigned long event, void *ptr)
9310 {
9311 	struct in_validator_info *ivi = (struct in_validator_info *) ptr;
9312 	struct net_device *dev = ivi->ivi_dev->dev;
9313 	struct mlxsw_sp *mlxsw_sp;
9314 	struct mlxsw_sp_rif *rif;
9315 	int err = 0;
9316 
9317 	mlxsw_sp = mlxsw_sp_lower_get(dev);
9318 	if (!mlxsw_sp)
9319 		return NOTIFY_DONE;
9320 
9321 	mutex_lock(&mlxsw_sp->router->lock);
9322 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9323 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9324 		goto out;
9325 
9326 	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false,
9327 					ivi->extack);
9328 out:
9329 	mutex_unlock(&mlxsw_sp->router->lock);
9330 	return notifier_from_errno(err);
9331 }
9332 
9333 struct mlxsw_sp_inet6addr_event_work {
9334 	struct work_struct work;
9335 	struct mlxsw_sp *mlxsw_sp;
9336 	struct net_device *dev;
9337 	netdevice_tracker dev_tracker;
9338 	unsigned long event;
9339 };
9340 
9341 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
9342 {
9343 	struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
9344 		container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
9345 	struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
9346 	struct net_device *dev = inet6addr_work->dev;
9347 	unsigned long event = inet6addr_work->event;
9348 	struct mlxsw_sp_rif *rif;
9349 
9350 	rtnl_lock();
9351 	mutex_lock(&mlxsw_sp->router->lock);
9352 
9353 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9354 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9355 		goto out;
9356 
9357 	__mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false, NULL);
9358 out:
9359 	mutex_unlock(&mlxsw_sp->router->lock);
9360 	rtnl_unlock();
9361 	netdev_put(dev, &inet6addr_work->dev_tracker);
9362 	kfree(inet6addr_work);
9363 }
9364 
9365 /* Called with rcu_read_lock() */
9366 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
9367 				    unsigned long event, void *ptr)
9368 {
9369 	struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
9370 	struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
9371 	struct net_device *dev = if6->idev->dev;
9372 	struct mlxsw_sp_router *router;
9373 
9374 	/* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
9375 	if (event == NETDEV_UP)
9376 		return NOTIFY_DONE;
9377 
9378 	inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
9379 	if (!inet6addr_work)
9380 		return NOTIFY_BAD;
9381 
9382 	router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
9383 	INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
9384 	inet6addr_work->mlxsw_sp = router->mlxsw_sp;
9385 	inet6addr_work->dev = dev;
9386 	inet6addr_work->event = event;
9387 	netdev_hold(dev, &inet6addr_work->dev_tracker, GFP_ATOMIC);
9388 	mlxsw_core_schedule_work(&inet6addr_work->work);
9389 
9390 	return NOTIFY_DONE;
9391 }
9392 
9393 static int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
9394 					  unsigned long event, void *ptr)
9395 {
9396 	struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
9397 	struct net_device *dev = i6vi->i6vi_dev->dev;
9398 	struct mlxsw_sp *mlxsw_sp;
9399 	struct mlxsw_sp_rif *rif;
9400 	int err = 0;
9401 
9402 	mlxsw_sp = mlxsw_sp_lower_get(dev);
9403 	if (!mlxsw_sp)
9404 		return NOTIFY_DONE;
9405 
9406 	mutex_lock(&mlxsw_sp->router->lock);
9407 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9408 	if (!mlxsw_sp_rif_should_config(rif, dev, event))
9409 		goto out;
9410 
9411 	err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, false,
9412 					i6vi->extack);
9413 out:
9414 	mutex_unlock(&mlxsw_sp->router->lock);
9415 	return notifier_from_errno(err);
9416 }
9417 
9418 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
9419 			     const char *mac, int mtu, u8 mac_profile)
9420 {
9421 	char ritr_pl[MLXSW_REG_RITR_LEN];
9422 	int err;
9423 
9424 	mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
9425 	err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9426 	if (err)
9427 		return err;
9428 
9429 	mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
9430 	mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
9431 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, mac_profile);
9432 	mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
9433 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
9434 }
9435 
9436 static int
9437 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
9438 				  struct mlxsw_sp_rif *rif,
9439 				  struct netlink_ext_ack *extack)
9440 {
9441 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
9442 	u8 old_mac_profile;
9443 	u16 fid_index;
9444 	int err;
9445 
9446 	fid_index = mlxsw_sp_fid_index(rif->fid);
9447 
9448 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
9449 	if (err)
9450 		return err;
9451 
9452 	old_mac_profile = rif->mac_profile_id;
9453 	err = mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, dev->dev_addr,
9454 					       extack);
9455 	if (err)
9456 		goto err_rif_mac_profile_replace;
9457 
9458 	err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
9459 				dev->mtu, rif->mac_profile_id);
9460 	if (err)
9461 		goto err_rif_edit;
9462 
9463 	err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
9464 	if (err)
9465 		goto err_rif_fdb_op;
9466 
9467 	if (rif->mtu != dev->mtu) {
9468 		struct mlxsw_sp_vr *vr;
9469 		int i;
9470 
9471 		/* The RIF is relevant only to its mr_table instance, as unlike
9472 		 * unicast routing, in multicast routing a RIF cannot be shared
9473 		 * between several multicast routing tables.
9474 		 */
9475 		vr = &mlxsw_sp->router->vrs[rif->vr_id];
9476 		for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
9477 			mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
9478 						   rif, dev->mtu);
9479 	}
9480 
9481 	ether_addr_copy(rif->addr, dev->dev_addr);
9482 	rif->mtu = dev->mtu;
9483 
9484 	netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
9485 
9486 	return 0;
9487 
9488 err_rif_fdb_op:
9489 	mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu,
9490 			  old_mac_profile);
9491 err_rif_edit:
9492 	mlxsw_sp_rif_mac_profile_replace(mlxsw_sp, rif, rif->addr, extack);
9493 err_rif_mac_profile_replace:
9494 	mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
9495 	return err;
9496 }
9497 
9498 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
9499 			    struct netdev_notifier_pre_changeaddr_info *info)
9500 {
9501 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
9502 	struct mlxsw_sp_rif_mac_profile *profile;
9503 	struct netlink_ext_ack *extack;
9504 	u8 max_rif_mac_profiles;
9505 	u64 occ;
9506 
9507 	extack = netdev_notifier_info_to_extack(&info->info);
9508 
9509 	profile = mlxsw_sp_rif_mac_profile_find(mlxsw_sp, info->dev_addr);
9510 	if (profile)
9511 		return 0;
9512 
9513 	max_rif_mac_profiles = mlxsw_sp->router->max_rif_mac_profile;
9514 	occ = mlxsw_sp_rif_mac_profiles_occ_get(mlxsw_sp);
9515 	if (occ < max_rif_mac_profiles)
9516 		return 0;
9517 
9518 	if (!mlxsw_sp_rif_mac_profile_is_shared(rif))
9519 		return 0;
9520 
9521 	NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interface MAC profiles");
9522 	return -ENOBUFS;
9523 }
9524 
9525 static bool mlxsw_sp_router_netdevice_interesting(struct mlxsw_sp *mlxsw_sp,
9526 						  struct net_device *dev)
9527 {
9528 	struct vlan_dev_priv *vlan;
9529 
9530 	if (netif_is_lag_master(dev) ||
9531 	    netif_is_bridge_master(dev) ||
9532 	    mlxsw_sp_port_dev_check(dev) ||
9533 	    mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev) ||
9534 	    netif_is_l3_master(dev))
9535 		return true;
9536 
9537 	if (!is_vlan_dev(dev))
9538 		return false;
9539 
9540 	vlan = vlan_dev_priv(dev);
9541 	return netif_is_lag_master(vlan->real_dev) ||
9542 	       netif_is_bridge_master(vlan->real_dev) ||
9543 	       mlxsw_sp_port_dev_check(vlan->real_dev);
9544 }
9545 
9546 static struct mlxsw_sp_crif *
9547 mlxsw_sp_crif_register(struct mlxsw_sp_router *router, struct net_device *dev)
9548 {
9549 	struct mlxsw_sp_crif *crif;
9550 	int err;
9551 
9552 	if (WARN_ON(mlxsw_sp_crif_lookup(router, dev)))
9553 		return NULL;
9554 
9555 	crif = mlxsw_sp_crif_alloc(dev);
9556 	if (!crif)
9557 		return ERR_PTR(-ENOMEM);
9558 
9559 	err = mlxsw_sp_crif_insert(router, crif);
9560 	if (err)
9561 		goto err_netdev_insert;
9562 
9563 	return crif;
9564 
9565 err_netdev_insert:
9566 	mlxsw_sp_crif_free(crif);
9567 	return ERR_PTR(err);
9568 }
9569 
9570 static void mlxsw_sp_crif_unregister(struct mlxsw_sp_router *router,
9571 				     struct mlxsw_sp_crif *crif)
9572 {
9573 	struct mlxsw_sp_nexthop *nh, *tmp;
9574 
9575 	mlxsw_sp_crif_remove(router, crif);
9576 
9577 	list_for_each_entry_safe(nh, tmp, &crif->nexthop_list, crif_list_node)
9578 		mlxsw_sp_nexthop_type_fini(router->mlxsw_sp, nh);
9579 
9580 	if (crif->rif)
9581 		crif->can_destroy = true;
9582 	else
9583 		mlxsw_sp_crif_free(crif);
9584 }
9585 
9586 static int mlxsw_sp_netdevice_register(struct mlxsw_sp_router *router,
9587 				       struct net_device *dev)
9588 {
9589 	struct mlxsw_sp_crif *crif;
9590 
9591 	if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev))
9592 		return 0;
9593 
9594 	crif = mlxsw_sp_crif_register(router, dev);
9595 	return PTR_ERR_OR_ZERO(crif);
9596 }
9597 
9598 static void mlxsw_sp_netdevice_unregister(struct mlxsw_sp_router *router,
9599 					  struct net_device *dev)
9600 {
9601 	struct mlxsw_sp_crif *crif;
9602 
9603 	if (!mlxsw_sp_router_netdevice_interesting(router->mlxsw_sp, dev))
9604 		return;
9605 
9606 	/* netdev_run_todo(), by way of netdev_wait_allrefs_any(), rebroadcasts
9607 	 * the NETDEV_UNREGISTER message, so we can get here twice. If that's
9608 	 * what happened, the netdevice state is NETREG_UNREGISTERED. In that
9609 	 * case, we expect to have collected the CRIF already, and warn if it
9610 	 * still exists. Otherwise we expect the CRIF to exist.
9611 	 */
9612 	crif = mlxsw_sp_crif_lookup(router, dev);
9613 	if (dev->reg_state == NETREG_UNREGISTERED) {
9614 		if (!WARN_ON(crif))
9615 			return;
9616 	}
9617 	if (WARN_ON(!crif))
9618 		return;
9619 
9620 	mlxsw_sp_crif_unregister(router, crif);
9621 }
9622 
9623 static bool mlxsw_sp_is_offload_xstats_event(unsigned long event)
9624 {
9625 	switch (event) {
9626 	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9627 	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9628 	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9629 	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9630 		return true;
9631 	}
9632 
9633 	return false;
9634 }
9635 
9636 static int
9637 mlxsw_sp_router_port_offload_xstats_cmd(struct mlxsw_sp_rif *rif,
9638 					unsigned long event,
9639 					struct netdev_notifier_offload_xstats_info *info)
9640 {
9641 	switch (info->type) {
9642 	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9643 		break;
9644 	default:
9645 		return 0;
9646 	}
9647 
9648 	switch (event) {
9649 	case NETDEV_OFFLOAD_XSTATS_ENABLE:
9650 		return mlxsw_sp_router_port_l3_stats_enable(rif);
9651 	case NETDEV_OFFLOAD_XSTATS_DISABLE:
9652 		mlxsw_sp_router_port_l3_stats_disable(rif);
9653 		return 0;
9654 	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:
9655 		mlxsw_sp_router_port_l3_stats_report_used(rif, info);
9656 		return 0;
9657 	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:
9658 		return mlxsw_sp_router_port_l3_stats_report_delta(rif, info);
9659 	}
9660 
9661 	WARN_ON_ONCE(1);
9662 	return 0;
9663 }
9664 
9665 static int
9666 mlxsw_sp_netdevice_offload_xstats_cmd(struct mlxsw_sp *mlxsw_sp,
9667 				      struct net_device *dev,
9668 				      unsigned long event,
9669 				      struct netdev_notifier_offload_xstats_info *info)
9670 {
9671 	struct mlxsw_sp_rif *rif;
9672 
9673 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9674 	if (!rif)
9675 		return 0;
9676 
9677 	return mlxsw_sp_router_port_offload_xstats_cmd(rif, event, info);
9678 }
9679 
9680 static bool mlxsw_sp_is_router_event(unsigned long event)
9681 {
9682 	switch (event) {
9683 	case NETDEV_PRE_CHANGEADDR:
9684 	case NETDEV_CHANGEADDR:
9685 	case NETDEV_CHANGEMTU:
9686 		return true;
9687 	default:
9688 		return false;
9689 	}
9690 }
9691 
9692 static int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
9693 						unsigned long event, void *ptr)
9694 {
9695 	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
9696 	struct mlxsw_sp *mlxsw_sp;
9697 	struct mlxsw_sp_rif *rif;
9698 
9699 	mlxsw_sp = mlxsw_sp_lower_get(dev);
9700 	if (!mlxsw_sp)
9701 		return 0;
9702 
9703 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
9704 	if (!rif)
9705 		return 0;
9706 
9707 	switch (event) {
9708 	case NETDEV_CHANGEMTU:
9709 	case NETDEV_CHANGEADDR:
9710 		return mlxsw_sp_router_port_change_event(mlxsw_sp, rif, extack);
9711 	case NETDEV_PRE_CHANGEADDR:
9712 		return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
9713 	default:
9714 		WARN_ON_ONCE(1);
9715 		break;
9716 	}
9717 
9718 	return 0;
9719 }
9720 
9721 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
9722 				  struct net_device *l3_dev,
9723 				  struct netlink_ext_ack *extack)
9724 {
9725 	struct mlxsw_sp_rif *rif;
9726 
9727 	/* If netdev is already associated with a RIF, then we need to
9728 	 * destroy it and create a new one with the new virtual router ID.
9729 	 */
9730 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9731 	if (rif)
9732 		__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false,
9733 					  extack);
9734 
9735 	return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, false,
9736 					 extack);
9737 }
9738 
9739 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
9740 				    struct net_device *l3_dev)
9741 {
9742 	struct mlxsw_sp_rif *rif;
9743 
9744 	rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
9745 	if (!rif)
9746 		return;
9747 	__mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, false, NULL);
9748 }
9749 
9750 static bool mlxsw_sp_is_vrf_event(unsigned long event, void *ptr)
9751 {
9752 	struct netdev_notifier_changeupper_info *info = ptr;
9753 
9754 	if (event != NETDEV_PRECHANGEUPPER && event != NETDEV_CHANGEUPPER)
9755 		return false;
9756 	return netif_is_l3_master(info->upper_dev);
9757 }
9758 
9759 static int
9760 mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
9761 			     struct netdev_notifier_changeupper_info *info)
9762 {
9763 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
9764 	int err = 0;
9765 
9766 	/* We do not create a RIF for a macvlan, but only use it to
9767 	 * direct more MAC addresses to the router.
9768 	 */
9769 	if (!mlxsw_sp || netif_is_macvlan(l3_dev))
9770 		return 0;
9771 
9772 	switch (event) {
9773 	case NETDEV_PRECHANGEUPPER:
9774 		break;
9775 	case NETDEV_CHANGEUPPER:
9776 		if (info->linking) {
9777 			struct netlink_ext_ack *extack;
9778 
9779 			extack = netdev_notifier_info_to_extack(&info->info);
9780 			err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
9781 		} else {
9782 			mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
9783 		}
9784 		break;
9785 	}
9786 
9787 	return err;
9788 }
9789 
9790 struct mlxsw_sp_router_replay_inetaddr_up {
9791 	struct mlxsw_sp *mlxsw_sp;
9792 	struct netlink_ext_ack *extack;
9793 	unsigned int done;
9794 	bool deslavement;
9795 };
9796 
9797 static int mlxsw_sp_router_replay_inetaddr_up(struct net_device *dev,
9798 					      struct netdev_nested_priv *priv)
9799 {
9800 	struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9801 	bool nomaster = ctx->deslavement;
9802 	struct mlxsw_sp_crif *crif;
9803 	int err;
9804 
9805 	if (mlxsw_sp_dev_addr_list_empty(dev))
9806 		return 0;
9807 
9808 	crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev);
9809 	if (!crif || crif->rif)
9810 		return 0;
9811 
9812 	if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP))
9813 		return 0;
9814 
9815 	err = __mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_UP,
9816 					nomaster, ctx->extack);
9817 	if (err)
9818 		return err;
9819 
9820 	ctx->done++;
9821 	return 0;
9822 }
9823 
9824 static int mlxsw_sp_router_unreplay_inetaddr_up(struct net_device *dev,
9825 						struct netdev_nested_priv *priv)
9826 {
9827 	struct mlxsw_sp_router_replay_inetaddr_up *ctx = priv->data;
9828 	bool nomaster = ctx->deslavement;
9829 	struct mlxsw_sp_crif *crif;
9830 
9831 	if (!ctx->done)
9832 		return 0;
9833 
9834 	if (mlxsw_sp_dev_addr_list_empty(dev))
9835 		return 0;
9836 
9837 	crif = mlxsw_sp_crif_lookup(ctx->mlxsw_sp->router, dev);
9838 	if (!crif || !crif->rif)
9839 		return 0;
9840 
9841 	/* We are rolling back NETDEV_UP, so ask for that. */
9842 	if (!mlxsw_sp_rif_should_config(crif->rif, dev, NETDEV_UP))
9843 		return 0;
9844 
9845 	__mlxsw_sp_inetaddr_event(ctx->mlxsw_sp, dev, NETDEV_DOWN, nomaster,
9846 				  NULL);
9847 
9848 	ctx->done--;
9849 	return 0;
9850 }
9851 
9852 int mlxsw_sp_netdevice_enslavement_replay(struct mlxsw_sp *mlxsw_sp,
9853 					  struct net_device *upper_dev,
9854 					  struct netlink_ext_ack *extack)
9855 {
9856 	struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9857 		.mlxsw_sp = mlxsw_sp,
9858 		.extack = extack,
9859 		.deslavement = false,
9860 	};
9861 	struct netdev_nested_priv priv = {
9862 		.data = &ctx,
9863 	};
9864 	int err;
9865 
9866 	err = mlxsw_sp_router_replay_inetaddr_up(upper_dev, &priv);
9867 	if (err)
9868 		return err;
9869 
9870 	err = netdev_walk_all_upper_dev_rcu(upper_dev,
9871 					    mlxsw_sp_router_replay_inetaddr_up,
9872 					    &priv);
9873 	if (err)
9874 		goto err_replay_up;
9875 
9876 	return 0;
9877 
9878 err_replay_up:
9879 	netdev_walk_all_upper_dev_rcu(upper_dev,
9880 				      mlxsw_sp_router_unreplay_inetaddr_up,
9881 				      &priv);
9882 	mlxsw_sp_router_unreplay_inetaddr_up(upper_dev, &priv);
9883 	return err;
9884 }
9885 
9886 void mlxsw_sp_netdevice_deslavement_replay(struct mlxsw_sp *mlxsw_sp,
9887 					   struct net_device *dev)
9888 {
9889 	struct mlxsw_sp_router_replay_inetaddr_up ctx = {
9890 		.mlxsw_sp = mlxsw_sp,
9891 		.deslavement = true,
9892 	};
9893 	struct netdev_nested_priv priv = {
9894 		.data = &ctx,
9895 	};
9896 
9897 	mlxsw_sp_router_replay_inetaddr_up(dev, &priv);
9898 }
9899 
9900 static int
9901 mlxsw_sp_port_vid_router_join_existing(struct mlxsw_sp_port *mlxsw_sp_port,
9902 				       u16 vid, struct net_device *dev,
9903 				       struct netlink_ext_ack *extack)
9904 {
9905 	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9906 
9907 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9908 							    vid);
9909 	if (WARN_ON(!mlxsw_sp_port_vlan))
9910 		return -EINVAL;
9911 
9912 	return mlxsw_sp_port_vlan_router_join_existing(mlxsw_sp_port_vlan,
9913 						       dev, extack);
9914 }
9915 
9916 static void
9917 mlxsw_sp_port_vid_router_leave(struct mlxsw_sp_port *mlxsw_sp_port, u16 vid,
9918 			       struct net_device *dev)
9919 {
9920 	struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
9921 
9922 	mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port,
9923 							    vid);
9924 	if (WARN_ON(!mlxsw_sp_port_vlan))
9925 		return;
9926 
9927 	__mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
9928 }
9929 
9930 static int __mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9931 					   struct net_device *lag_dev,
9932 					   struct netlink_ext_ack *extack)
9933 {
9934 	u16 default_vid = MLXSW_SP_DEFAULT_VID;
9935 	struct net_device *upper_dev;
9936 	struct list_head *iter;
9937 	int done = 0;
9938 	u16 vid;
9939 	int err;
9940 
9941 	err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, default_vid,
9942 						     lag_dev, extack);
9943 	if (err)
9944 		return err;
9945 
9946 	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9947 		if (!is_vlan_dev(upper_dev))
9948 			continue;
9949 
9950 		vid = vlan_dev_vlan_id(upper_dev);
9951 		err = mlxsw_sp_port_vid_router_join_existing(mlxsw_sp_port, vid,
9952 							     upper_dev, extack);
9953 		if (err)
9954 			goto err_router_join_dev;
9955 
9956 		++done;
9957 	}
9958 
9959 	return 0;
9960 
9961 err_router_join_dev:
9962 	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9963 		if (!is_vlan_dev(upper_dev))
9964 			continue;
9965 		if (!done--)
9966 			break;
9967 
9968 		vid = vlan_dev_vlan_id(upper_dev);
9969 		mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev);
9970 	}
9971 
9972 	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev);
9973 	return err;
9974 }
9975 
9976 static void
9977 __mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9978 				 struct net_device *lag_dev)
9979 {
9980 	u16 default_vid = MLXSW_SP_DEFAULT_VID;
9981 	struct net_device *upper_dev;
9982 	struct list_head *iter;
9983 	u16 vid;
9984 
9985 	netdev_for_each_upper_dev_rcu(lag_dev, upper_dev, iter) {
9986 		if (!is_vlan_dev(upper_dev))
9987 			continue;
9988 
9989 		vid = vlan_dev_vlan_id(upper_dev);
9990 		mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, vid, upper_dev);
9991 	}
9992 
9993 	mlxsw_sp_port_vid_router_leave(mlxsw_sp_port, default_vid, lag_dev);
9994 }
9995 
9996 int mlxsw_sp_router_port_join_lag(struct mlxsw_sp_port *mlxsw_sp_port,
9997 				  struct net_device *lag_dev,
9998 				  struct netlink_ext_ack *extack)
9999 {
10000 	int err;
10001 
10002 	mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10003 	err = __mlxsw_sp_router_port_join_lag(mlxsw_sp_port, lag_dev, extack);
10004 	mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10005 
10006 	return err;
10007 }
10008 
10009 void mlxsw_sp_router_port_leave_lag(struct mlxsw_sp_port *mlxsw_sp_port,
10010 				    struct net_device *lag_dev)
10011 {
10012 	mutex_lock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10013 	__mlxsw_sp_router_port_leave_lag(mlxsw_sp_port, lag_dev);
10014 	mutex_unlock(&mlxsw_sp_port->mlxsw_sp->router->lock);
10015 }
10016 
10017 static int mlxsw_sp_router_netdevice_event(struct notifier_block *nb,
10018 					   unsigned long event, void *ptr)
10019 {
10020 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
10021 	struct mlxsw_sp_router *router;
10022 	struct mlxsw_sp *mlxsw_sp;
10023 	int err = 0;
10024 
10025 	router = container_of(nb, struct mlxsw_sp_router, netdevice_nb);
10026 	mlxsw_sp = router->mlxsw_sp;
10027 
10028 	mutex_lock(&mlxsw_sp->router->lock);
10029 
10030 	if (event == NETDEV_REGISTER) {
10031 		err = mlxsw_sp_netdevice_register(router, dev);
10032 		if (err)
10033 			/* No need to roll this back, UNREGISTER will collect it
10034 			 * anyhow.
10035 			 */
10036 			goto out;
10037 	}
10038 
10039 	if (mlxsw_sp_is_offload_xstats_event(event))
10040 		err = mlxsw_sp_netdevice_offload_xstats_cmd(mlxsw_sp, dev,
10041 							    event, ptr);
10042 	else if (mlxsw_sp_netdev_is_ipip_ol(mlxsw_sp, dev))
10043 		err = mlxsw_sp_netdevice_ipip_ol_event(mlxsw_sp, dev,
10044 						       event, ptr);
10045 	else if (mlxsw_sp_netdev_is_ipip_ul(mlxsw_sp, dev))
10046 		err = mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, dev,
10047 						       event, ptr);
10048 	else if (mlxsw_sp_is_router_event(event))
10049 		err = mlxsw_sp_netdevice_router_port_event(dev, event, ptr);
10050 	else if (mlxsw_sp_is_vrf_event(event, ptr))
10051 		err = mlxsw_sp_netdevice_vrf_event(dev, event, ptr);
10052 
10053 	if (event == NETDEV_UNREGISTER)
10054 		mlxsw_sp_netdevice_unregister(router, dev);
10055 
10056 out:
10057 	mutex_unlock(&mlxsw_sp->router->lock);
10058 
10059 	return notifier_from_errno(err);
10060 }
10061 
10062 struct mlxsw_sp_macvlan_replay {
10063 	struct mlxsw_sp *mlxsw_sp;
10064 	struct netlink_ext_ack *extack;
10065 };
10066 
10067 static int mlxsw_sp_macvlan_replay_upper(struct net_device *dev,
10068 					 struct netdev_nested_priv *priv)
10069 {
10070 	const struct mlxsw_sp_macvlan_replay *rms = priv->data;
10071 	struct netlink_ext_ack *extack = rms->extack;
10072 	struct mlxsw_sp *mlxsw_sp = rms->mlxsw_sp;
10073 
10074 	if (!netif_is_macvlan(dev))
10075 		return 0;
10076 
10077 	return mlxsw_sp_rif_macvlan_add(mlxsw_sp, dev, extack);
10078 }
10079 
10080 static int mlxsw_sp_macvlan_replay(struct mlxsw_sp_rif *rif,
10081 				   struct netlink_ext_ack *extack)
10082 {
10083 	struct mlxsw_sp_macvlan_replay rms = {
10084 		.mlxsw_sp = rif->mlxsw_sp,
10085 		.extack = extack,
10086 	};
10087 	struct netdev_nested_priv priv = {
10088 		.data = &rms,
10089 	};
10090 
10091 	return netdev_walk_all_upper_dev_rcu(mlxsw_sp_rif_dev(rif),
10092 					     mlxsw_sp_macvlan_replay_upper,
10093 					     &priv);
10094 }
10095 
10096 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev,
10097 					struct netdev_nested_priv *priv)
10098 {
10099 	struct mlxsw_sp_rif *rif = (struct mlxsw_sp_rif *)priv->data;
10100 
10101 	if (!netif_is_macvlan(dev))
10102 		return 0;
10103 
10104 	return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10105 				   mlxsw_sp_fid_index(rif->fid), false);
10106 }
10107 
10108 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
10109 {
10110 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10111 	struct netdev_nested_priv priv = {
10112 		.data = (void *)rif,
10113 	};
10114 
10115 	if (!netif_is_macvlan_port(dev))
10116 		return 0;
10117 
10118 	return netdev_walk_all_upper_dev_rcu(dev,
10119 					     __mlxsw_sp_rif_macvlan_flush, &priv);
10120 }
10121 
10122 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
10123 				       const struct mlxsw_sp_rif_params *params)
10124 {
10125 	struct mlxsw_sp_rif_subport *rif_subport;
10126 
10127 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
10128 	refcount_set(&rif_subport->ref_count, 1);
10129 	rif_subport->vid = params->vid;
10130 	rif_subport->lag = params->lag;
10131 	if (params->lag)
10132 		rif_subport->lag_id = params->lag_id;
10133 	else
10134 		rif_subport->system_port = params->system_port;
10135 }
10136 
10137 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
10138 {
10139 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10140 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10141 	struct mlxsw_sp_rif_subport *rif_subport;
10142 	char ritr_pl[MLXSW_REG_RITR_LEN];
10143 	u16 efid;
10144 
10145 	rif_subport = mlxsw_sp_rif_subport_rif(rif);
10146 	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
10147 			    rif->rif_index, rif->vr_id, dev->mtu);
10148 	mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr);
10149 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
10150 	efid = mlxsw_sp_fid_index(rif->fid);
10151 	mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
10152 				  rif_subport->lag ? rif_subport->lag_id :
10153 						     rif_subport->system_port,
10154 				  efid, 0);
10155 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10156 }
10157 
10158 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif,
10159 					  struct netlink_ext_ack *extack)
10160 {
10161 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10162 	u8 mac_profile;
10163 	int err;
10164 
10165 	err = mlxsw_sp_rif_mac_profile_get(rif->mlxsw_sp, rif->addr,
10166 					   &mac_profile, extack);
10167 	if (err)
10168 		return err;
10169 	rif->mac_profile_id = mac_profile;
10170 
10171 	err = mlxsw_sp_rif_subport_op(rif, true);
10172 	if (err)
10173 		goto err_rif_subport_op;
10174 
10175 	err = mlxsw_sp_macvlan_replay(rif, extack);
10176 	if (err)
10177 		goto err_macvlan_replay;
10178 
10179 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10180 				  mlxsw_sp_fid_index(rif->fid), true);
10181 	if (err)
10182 		goto err_rif_fdb_op;
10183 
10184 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10185 	if (err)
10186 		goto err_fid_rif_set;
10187 
10188 	return 0;
10189 
10190 err_fid_rif_set:
10191 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10192 			    mlxsw_sp_fid_index(rif->fid), false);
10193 err_rif_fdb_op:
10194 	mlxsw_sp_rif_macvlan_flush(rif);
10195 err_macvlan_replay:
10196 	mlxsw_sp_rif_subport_op(rif, false);
10197 err_rif_subport_op:
10198 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, mac_profile);
10199 	return err;
10200 }
10201 
10202 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
10203 {
10204 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10205 	struct mlxsw_sp_fid *fid = rif->fid;
10206 
10207 	mlxsw_sp_fid_rif_unset(fid);
10208 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10209 			    mlxsw_sp_fid_index(fid), false);
10210 	mlxsw_sp_rif_macvlan_flush(rif);
10211 	mlxsw_sp_rif_subport_op(rif, false);
10212 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10213 }
10214 
10215 static struct mlxsw_sp_fid *
10216 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
10217 			     const struct mlxsw_sp_rif_params *params,
10218 			     struct netlink_ext_ack *extack)
10219 {
10220 	return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
10221 }
10222 
10223 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
10224 	.type			= MLXSW_SP_RIF_TYPE_SUBPORT,
10225 	.rif_size		= sizeof(struct mlxsw_sp_rif_subport),
10226 	.setup			= mlxsw_sp_rif_subport_setup,
10227 	.configure		= mlxsw_sp_rif_subport_configure,
10228 	.deconfigure		= mlxsw_sp_rif_subport_deconfigure,
10229 	.fid_get		= mlxsw_sp_rif_subport_fid_get,
10230 };
10231 
10232 static int mlxsw_sp_rif_fid_op(struct mlxsw_sp_rif *rif, u16 fid, bool enable)
10233 {
10234 	enum mlxsw_reg_ritr_if_type type = MLXSW_REG_RITR_FID_IF;
10235 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10236 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10237 	char ritr_pl[MLXSW_REG_RITR_LEN];
10238 
10239 	mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
10240 			    dev->mtu);
10241 	mlxsw_reg_ritr_mac_pack(ritr_pl, dev->dev_addr);
10242 	mlxsw_reg_ritr_if_mac_profile_id_set(ritr_pl, rif->mac_profile_id);
10243 	mlxsw_reg_ritr_fid_if_fid_set(ritr_pl, fid);
10244 
10245 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10246 }
10247 
10248 u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
10249 {
10250 	return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
10251 }
10252 
10253 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif,
10254 				      struct netlink_ext_ack *extack)
10255 {
10256 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10257 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10258 	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
10259 	u8 mac_profile;
10260 	int err;
10261 
10262 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
10263 					   &mac_profile, extack);
10264 	if (err)
10265 		return err;
10266 	rif->mac_profile_id = mac_profile;
10267 
10268 	err = mlxsw_sp_rif_fid_op(rif, fid_index, true);
10269 	if (err)
10270 		goto err_rif_fid_op;
10271 
10272 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10273 				     mlxsw_sp_router_port(mlxsw_sp), true);
10274 	if (err)
10275 		goto err_fid_mc_flood_set;
10276 
10277 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10278 				     mlxsw_sp_router_port(mlxsw_sp), true);
10279 	if (err)
10280 		goto err_fid_bc_flood_set;
10281 
10282 	err = mlxsw_sp_macvlan_replay(rif, extack);
10283 	if (err)
10284 		goto err_macvlan_replay;
10285 
10286 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10287 				  mlxsw_sp_fid_index(rif->fid), true);
10288 	if (err)
10289 		goto err_rif_fdb_op;
10290 
10291 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10292 	if (err)
10293 		goto err_fid_rif_set;
10294 
10295 	return 0;
10296 
10297 err_fid_rif_set:
10298 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10299 			    mlxsw_sp_fid_index(rif->fid), false);
10300 err_rif_fdb_op:
10301 	mlxsw_sp_rif_macvlan_flush(rif);
10302 err_macvlan_replay:
10303 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10304 			       mlxsw_sp_router_port(mlxsw_sp), false);
10305 err_fid_bc_flood_set:
10306 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10307 			       mlxsw_sp_router_port(mlxsw_sp), false);
10308 err_fid_mc_flood_set:
10309 	mlxsw_sp_rif_fid_op(rif, fid_index, false);
10310 err_rif_fid_op:
10311 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10312 	return err;
10313 }
10314 
10315 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
10316 {
10317 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10318 	u16 fid_index = mlxsw_sp_fid_index(rif->fid);
10319 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10320 	struct mlxsw_sp_fid *fid = rif->fid;
10321 
10322 	mlxsw_sp_fid_rif_unset(fid);
10323 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10324 			    mlxsw_sp_fid_index(fid), false);
10325 	mlxsw_sp_rif_macvlan_flush(rif);
10326 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10327 			       mlxsw_sp_router_port(mlxsw_sp), false);
10328 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10329 			       mlxsw_sp_router_port(mlxsw_sp), false);
10330 	mlxsw_sp_rif_fid_op(rif, fid_index, false);
10331 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10332 }
10333 
10334 static struct mlxsw_sp_fid *
10335 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
10336 			 const struct mlxsw_sp_rif_params *params,
10337 			 struct netlink_ext_ack *extack)
10338 {
10339 	int rif_ifindex = mlxsw_sp_rif_dev_ifindex(rif);
10340 
10341 	return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif_ifindex);
10342 }
10343 
10344 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10345 {
10346 	struct switchdev_notifier_fdb_info info = {};
10347 	struct net_device *dev;
10348 
10349 	dev = br_fdb_find_port(mlxsw_sp_rif_dev(rif), mac, 0);
10350 	if (!dev)
10351 		return;
10352 
10353 	info.addr = mac;
10354 	info.vid = 0;
10355 	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
10356 				 NULL);
10357 }
10358 
10359 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
10360 	.type			= MLXSW_SP_RIF_TYPE_FID,
10361 	.rif_size		= sizeof(struct mlxsw_sp_rif),
10362 	.configure		= mlxsw_sp_rif_fid_configure,
10363 	.deconfigure		= mlxsw_sp_rif_fid_deconfigure,
10364 	.fid_get		= mlxsw_sp_rif_fid_fid_get,
10365 	.fdb_del		= mlxsw_sp_rif_fid_fdb_del,
10366 };
10367 
10368 static struct mlxsw_sp_fid *
10369 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
10370 			  const struct mlxsw_sp_rif_params *params,
10371 			  struct netlink_ext_ack *extack)
10372 {
10373 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10374 	struct net_device *br_dev;
10375 
10376 	if (WARN_ON(!params->vid))
10377 		return ERR_PTR(-EINVAL);
10378 
10379 	if (is_vlan_dev(dev)) {
10380 		br_dev = vlan_dev_real_dev(dev);
10381 		if (WARN_ON(!netif_is_bridge_master(br_dev)))
10382 			return ERR_PTR(-EINVAL);
10383 	}
10384 
10385 	return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, params->vid);
10386 }
10387 
10388 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
10389 {
10390 	struct net_device *rif_dev = mlxsw_sp_rif_dev(rif);
10391 	struct switchdev_notifier_fdb_info info = {};
10392 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10393 	struct net_device *br_dev;
10394 	struct net_device *dev;
10395 
10396 	br_dev = is_vlan_dev(rif_dev) ? vlan_dev_real_dev(rif_dev) : rif_dev;
10397 	dev = br_fdb_find_port(br_dev, mac, vid);
10398 	if (!dev)
10399 		return;
10400 
10401 	info.addr = mac;
10402 	info.vid = vid;
10403 	call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
10404 				 NULL);
10405 }
10406 
10407 static int mlxsw_sp_rif_vlan_op(struct mlxsw_sp_rif *rif, u16 vid, u16 efid,
10408 				bool enable)
10409 {
10410 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10411 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10412 	char ritr_pl[MLXSW_REG_RITR_LEN];
10413 
10414 	mlxsw_reg_ritr_vlan_if_pack(ritr_pl, enable, rif->rif_index, rif->vr_id,
10415 				    dev->mtu, dev->dev_addr,
10416 				    rif->mac_profile_id, vid, efid);
10417 
10418 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10419 }
10420 
10421 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif, u16 efid,
10422 				       struct netlink_ext_ack *extack)
10423 {
10424 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10425 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10426 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10427 	u8 mac_profile;
10428 	int err;
10429 
10430 	err = mlxsw_sp_rif_mac_profile_get(mlxsw_sp, rif->addr,
10431 					   &mac_profile, extack);
10432 	if (err)
10433 		return err;
10434 	rif->mac_profile_id = mac_profile;
10435 
10436 	err = mlxsw_sp_rif_vlan_op(rif, vid, efid, true);
10437 	if (err)
10438 		goto err_rif_vlan_fid_op;
10439 
10440 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10441 				     mlxsw_sp_router_port(mlxsw_sp), true);
10442 	if (err)
10443 		goto err_fid_mc_flood_set;
10444 
10445 	err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10446 				     mlxsw_sp_router_port(mlxsw_sp), true);
10447 	if (err)
10448 		goto err_fid_bc_flood_set;
10449 
10450 	err = mlxsw_sp_macvlan_replay(rif, extack);
10451 	if (err)
10452 		goto err_macvlan_replay;
10453 
10454 	err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10455 				  mlxsw_sp_fid_index(rif->fid), true);
10456 	if (err)
10457 		goto err_rif_fdb_op;
10458 
10459 	err = mlxsw_sp_fid_rif_set(rif->fid, rif);
10460 	if (err)
10461 		goto err_fid_rif_set;
10462 
10463 	return 0;
10464 
10465 err_fid_rif_set:
10466 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10467 			    mlxsw_sp_fid_index(rif->fid), false);
10468 err_rif_fdb_op:
10469 	mlxsw_sp_rif_macvlan_flush(rif);
10470 err_macvlan_replay:
10471 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10472 			       mlxsw_sp_router_port(mlxsw_sp), false);
10473 err_fid_bc_flood_set:
10474 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10475 			       mlxsw_sp_router_port(mlxsw_sp), false);
10476 err_fid_mc_flood_set:
10477 	mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
10478 err_rif_vlan_fid_op:
10479 	mlxsw_sp_rif_mac_profile_put(mlxsw_sp, mac_profile);
10480 	return err;
10481 }
10482 
10483 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
10484 {
10485 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10486 	u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
10487 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10488 
10489 	mlxsw_sp_fid_rif_unset(rif->fid);
10490 	mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
10491 			    mlxsw_sp_fid_index(rif->fid), false);
10492 	mlxsw_sp_rif_macvlan_flush(rif);
10493 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
10494 			       mlxsw_sp_router_port(mlxsw_sp), false);
10495 	mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
10496 			       mlxsw_sp_router_port(mlxsw_sp), false);
10497 	mlxsw_sp_rif_vlan_op(rif, vid, 0, false);
10498 	mlxsw_sp_rif_mac_profile_put(rif->mlxsw_sp, rif->mac_profile_id);
10499 }
10500 
10501 static int mlxsw_sp1_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10502 					struct netlink_ext_ack *extack)
10503 {
10504 	return mlxsw_sp_rif_vlan_configure(rif, 0, extack);
10505 }
10506 
10507 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_vlan_ops = {
10508 	.type			= MLXSW_SP_RIF_TYPE_VLAN,
10509 	.rif_size		= sizeof(struct mlxsw_sp_rif),
10510 	.configure		= mlxsw_sp1_rif_vlan_configure,
10511 	.deconfigure		= mlxsw_sp_rif_vlan_deconfigure,
10512 	.fid_get		= mlxsw_sp_rif_vlan_fid_get,
10513 	.fdb_del		= mlxsw_sp_rif_vlan_fdb_del,
10514 };
10515 
10516 static int mlxsw_sp2_rif_vlan_configure(struct mlxsw_sp_rif *rif,
10517 					struct netlink_ext_ack *extack)
10518 {
10519 	u16 efid = mlxsw_sp_fid_index(rif->fid);
10520 
10521 	return mlxsw_sp_rif_vlan_configure(rif, efid, extack);
10522 }
10523 
10524 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_vlan_ops = {
10525 	.type			= MLXSW_SP_RIF_TYPE_VLAN,
10526 	.rif_size		= sizeof(struct mlxsw_sp_rif),
10527 	.configure		= mlxsw_sp2_rif_vlan_configure,
10528 	.deconfigure		= mlxsw_sp_rif_vlan_deconfigure,
10529 	.fid_get		= mlxsw_sp_rif_vlan_fid_get,
10530 	.fdb_del		= mlxsw_sp_rif_vlan_fdb_del,
10531 };
10532 
10533 static struct mlxsw_sp_rif_ipip_lb *
10534 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
10535 {
10536 	return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
10537 }
10538 
10539 static void
10540 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
10541 			   const struct mlxsw_sp_rif_params *params)
10542 {
10543 	struct mlxsw_sp_rif_params_ipip_lb *params_lb;
10544 	struct mlxsw_sp_rif_ipip_lb *rif_lb;
10545 
10546 	params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
10547 				 common);
10548 	rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
10549 	rif_lb->lb_config = params_lb->lb_config;
10550 }
10551 
10552 static int
10553 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10554 				struct netlink_ext_ack *extack)
10555 {
10556 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10557 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10558 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
10559 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10560 	struct mlxsw_sp_vr *ul_vr;
10561 	int err;
10562 
10563 	ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, extack);
10564 	if (IS_ERR(ul_vr))
10565 		return PTR_ERR(ul_vr);
10566 
10567 	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
10568 	if (err)
10569 		goto err_loopback_op;
10570 
10571 	lb_rif->ul_vr_id = ul_vr->id;
10572 	lb_rif->ul_rif_id = 0;
10573 	++ul_vr->rif_count;
10574 	return 0;
10575 
10576 err_loopback_op:
10577 	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
10578 	return err;
10579 }
10580 
10581 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10582 {
10583 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10584 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10585 	struct mlxsw_sp_vr *ul_vr;
10586 
10587 	ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
10588 	mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
10589 
10590 	--ul_vr->rif_count;
10591 	mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
10592 }
10593 
10594 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
10595 	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
10596 	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
10597 	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
10598 	.configure		= mlxsw_sp1_rif_ipip_lb_configure,
10599 	.deconfigure		= mlxsw_sp1_rif_ipip_lb_deconfigure,
10600 };
10601 
10602 static const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
10603 	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
10604 	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp1_rif_vlan_ops,
10605 	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
10606 	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp1_rif_ipip_lb_ops,
10607 };
10608 
10609 static int
10610 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
10611 {
10612 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10613 	char ritr_pl[MLXSW_REG_RITR_LEN];
10614 
10615 	mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
10616 			    ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
10617 	mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
10618 					     MLXSW_REG_RITR_LOOPBACK_GENERIC);
10619 
10620 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
10621 }
10622 
10623 static struct mlxsw_sp_rif *
10624 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
10625 		       struct mlxsw_sp_crif *ul_crif,
10626 		       struct netlink_ext_ack *extack)
10627 {
10628 	struct mlxsw_sp_rif *ul_rif;
10629 	u8 rif_entries = 1;
10630 	u16 rif_index;
10631 	int err;
10632 
10633 	err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index, rif_entries);
10634 	if (err) {
10635 		NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
10636 		return ERR_PTR(err);
10637 	}
10638 
10639 	ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id,
10640 				    ul_crif);
10641 	if (!ul_rif) {
10642 		err = -ENOMEM;
10643 		goto err_rif_alloc;
10644 	}
10645 
10646 	mlxsw_sp->router->rifs[rif_index] = ul_rif;
10647 	ul_rif->mlxsw_sp = mlxsw_sp;
10648 	ul_rif->rif_entries = rif_entries;
10649 	err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
10650 	if (err)
10651 		goto ul_rif_op_err;
10652 
10653 	atomic_add(rif_entries, &mlxsw_sp->router->rifs_count);
10654 	return ul_rif;
10655 
10656 ul_rif_op_err:
10657 	mlxsw_sp->router->rifs[rif_index] = NULL;
10658 	mlxsw_sp_rif_free(ul_rif);
10659 err_rif_alloc:
10660 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10661 	return ERR_PTR(err);
10662 }
10663 
10664 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
10665 {
10666 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10667 	u8 rif_entries = ul_rif->rif_entries;
10668 	u16 rif_index = ul_rif->rif_index;
10669 
10670 	atomic_sub(rif_entries, &mlxsw_sp->router->rifs_count);
10671 	mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
10672 	mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
10673 	mlxsw_sp_rif_free(ul_rif);
10674 	mlxsw_sp_rif_index_free(mlxsw_sp, rif_index, rif_entries);
10675 }
10676 
10677 static struct mlxsw_sp_rif *
10678 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
10679 		    struct mlxsw_sp_crif *ul_crif,
10680 		    struct netlink_ext_ack *extack)
10681 {
10682 	struct mlxsw_sp_vr *vr;
10683 	int err;
10684 
10685 	vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
10686 	if (IS_ERR(vr))
10687 		return ERR_CAST(vr);
10688 
10689 	if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
10690 		return vr->ul_rif;
10691 
10692 	vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, ul_crif, extack);
10693 	if (IS_ERR(vr->ul_rif)) {
10694 		err = PTR_ERR(vr->ul_rif);
10695 		goto err_ul_rif_create;
10696 	}
10697 
10698 	vr->rif_count++;
10699 	refcount_set(&vr->ul_rif_refcnt, 1);
10700 
10701 	return vr->ul_rif;
10702 
10703 err_ul_rif_create:
10704 	mlxsw_sp_vr_put(mlxsw_sp, vr);
10705 	return ERR_PTR(err);
10706 }
10707 
10708 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
10709 {
10710 	struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
10711 	struct mlxsw_sp_vr *vr;
10712 
10713 	vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
10714 
10715 	if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
10716 		return;
10717 
10718 	vr->rif_count--;
10719 	mlxsw_sp_ul_rif_destroy(ul_rif);
10720 	mlxsw_sp_vr_put(mlxsw_sp, vr);
10721 }
10722 
10723 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
10724 			       u16 *ul_rif_index)
10725 {
10726 	struct mlxsw_sp_rif *ul_rif;
10727 	int err = 0;
10728 
10729 	mutex_lock(&mlxsw_sp->router->lock);
10730 	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, NULL);
10731 	if (IS_ERR(ul_rif)) {
10732 		err = PTR_ERR(ul_rif);
10733 		goto out;
10734 	}
10735 	*ul_rif_index = ul_rif->rif_index;
10736 out:
10737 	mutex_unlock(&mlxsw_sp->router->lock);
10738 	return err;
10739 }
10740 
10741 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
10742 {
10743 	struct mlxsw_sp_rif *ul_rif;
10744 
10745 	mutex_lock(&mlxsw_sp->router->lock);
10746 	ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
10747 	if (WARN_ON(!ul_rif))
10748 		goto out;
10749 
10750 	mlxsw_sp_ul_rif_put(ul_rif);
10751 out:
10752 	mutex_unlock(&mlxsw_sp->router->lock);
10753 }
10754 
10755 static int
10756 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif,
10757 				struct netlink_ext_ack *extack)
10758 {
10759 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10760 	struct net_device *dev = mlxsw_sp_rif_dev(rif);
10761 	u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(dev);
10762 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10763 	struct mlxsw_sp_rif *ul_rif;
10764 	int err;
10765 
10766 	ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL, extack);
10767 	if (IS_ERR(ul_rif))
10768 		return PTR_ERR(ul_rif);
10769 
10770 	err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
10771 	if (err)
10772 		goto err_loopback_op;
10773 
10774 	lb_rif->ul_vr_id = 0;
10775 	lb_rif->ul_rif_id = ul_rif->rif_index;
10776 
10777 	return 0;
10778 
10779 err_loopback_op:
10780 	mlxsw_sp_ul_rif_put(ul_rif);
10781 	return err;
10782 }
10783 
10784 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
10785 {
10786 	struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
10787 	struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
10788 	struct mlxsw_sp_rif *ul_rif;
10789 
10790 	ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
10791 	mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
10792 	mlxsw_sp_ul_rif_put(ul_rif);
10793 }
10794 
10795 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
10796 	.type			= MLXSW_SP_RIF_TYPE_IPIP_LB,
10797 	.rif_size		= sizeof(struct mlxsw_sp_rif_ipip_lb),
10798 	.setup                  = mlxsw_sp_rif_ipip_lb_setup,
10799 	.configure		= mlxsw_sp2_rif_ipip_lb_configure,
10800 	.deconfigure		= mlxsw_sp2_rif_ipip_lb_deconfigure,
10801 };
10802 
10803 static const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
10804 	[MLXSW_SP_RIF_TYPE_SUBPORT]	= &mlxsw_sp_rif_subport_ops,
10805 	[MLXSW_SP_RIF_TYPE_VLAN]	= &mlxsw_sp2_rif_vlan_ops,
10806 	[MLXSW_SP_RIF_TYPE_FID]		= &mlxsw_sp_rif_fid_ops,
10807 	[MLXSW_SP_RIF_TYPE_IPIP_LB]	= &mlxsw_sp2_rif_ipip_lb_ops,
10808 };
10809 
10810 static int mlxsw_sp_rifs_table_init(struct mlxsw_sp *mlxsw_sp)
10811 {
10812 	struct gen_pool *rifs_table;
10813 	int err;
10814 
10815 	rifs_table = gen_pool_create(0, -1);
10816 	if (!rifs_table)
10817 		return -ENOMEM;
10818 
10819 	gen_pool_set_algo(rifs_table, gen_pool_first_fit_order_align,
10820 			  NULL);
10821 
10822 	err = gen_pool_add(rifs_table, MLXSW_SP_ROUTER_GENALLOC_OFFSET,
10823 			   MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS), -1);
10824 	if (err)
10825 		goto err_gen_pool_add;
10826 
10827 	mlxsw_sp->router->rifs_table = rifs_table;
10828 
10829 	return 0;
10830 
10831 err_gen_pool_add:
10832 	gen_pool_destroy(rifs_table);
10833 	return err;
10834 }
10835 
10836 static void mlxsw_sp_rifs_table_fini(struct mlxsw_sp *mlxsw_sp)
10837 {
10838 	gen_pool_destroy(mlxsw_sp->router->rifs_table);
10839 }
10840 
10841 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
10842 {
10843 	u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10844 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10845 	struct mlxsw_core *core = mlxsw_sp->core;
10846 	int err;
10847 
10848 	if (!MLXSW_CORE_RES_VALID(core, MAX_RIF_MAC_PROFILES))
10849 		return -EIO;
10850 	mlxsw_sp->router->max_rif_mac_profile =
10851 		MLXSW_CORE_RES_GET(core, MAX_RIF_MAC_PROFILES);
10852 
10853 	mlxsw_sp->router->rifs = kcalloc(max_rifs,
10854 					 sizeof(struct mlxsw_sp_rif *),
10855 					 GFP_KERNEL);
10856 	if (!mlxsw_sp->router->rifs)
10857 		return -ENOMEM;
10858 
10859 	err = mlxsw_sp_rifs_table_init(mlxsw_sp);
10860 	if (err)
10861 		goto err_rifs_table_init;
10862 
10863 	idr_init(&mlxsw_sp->router->rif_mac_profiles_idr);
10864 	atomic_set(&mlxsw_sp->router->rif_mac_profiles_count, 0);
10865 	atomic_set(&mlxsw_sp->router->rifs_count, 0);
10866 	devl_resource_occ_get_register(devlink,
10867 				       MLXSW_SP_RESOURCE_RIF_MAC_PROFILES,
10868 				       mlxsw_sp_rif_mac_profiles_occ_get,
10869 				       mlxsw_sp);
10870 	devl_resource_occ_get_register(devlink,
10871 				       MLXSW_SP_RESOURCE_RIFS,
10872 				       mlxsw_sp_rifs_occ_get,
10873 				       mlxsw_sp);
10874 
10875 	return 0;
10876 
10877 err_rifs_table_init:
10878 	kfree(mlxsw_sp->router->rifs);
10879 	return err;
10880 }
10881 
10882 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
10883 {
10884 	int max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
10885 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
10886 	int i;
10887 
10888 	WARN_ON_ONCE(atomic_read(&mlxsw_sp->router->rifs_count));
10889 	for (i = 0; i < max_rifs; i++)
10890 		WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
10891 
10892 	devl_resource_occ_get_unregister(devlink, MLXSW_SP_RESOURCE_RIFS);
10893 	devl_resource_occ_get_unregister(devlink,
10894 					 MLXSW_SP_RESOURCE_RIF_MAC_PROFILES);
10895 	WARN_ON(!idr_is_empty(&mlxsw_sp->router->rif_mac_profiles_idr));
10896 	idr_destroy(&mlxsw_sp->router->rif_mac_profiles_idr);
10897 	mlxsw_sp_rifs_table_fini(mlxsw_sp);
10898 	kfree(mlxsw_sp->router->rifs);
10899 }
10900 
10901 static int
10902 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
10903 {
10904 	char tigcr_pl[MLXSW_REG_TIGCR_LEN];
10905 
10906 	mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
10907 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
10908 }
10909 
10910 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
10911 {
10912 	int err;
10913 
10914 	INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
10915 
10916 	err = mlxsw_sp_ipip_ecn_encap_init(mlxsw_sp);
10917 	if (err)
10918 		return err;
10919 	err = mlxsw_sp_ipip_ecn_decap_init(mlxsw_sp);
10920 	if (err)
10921 		return err;
10922 
10923 	return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
10924 }
10925 
10926 static int mlxsw_sp1_ipips_init(struct mlxsw_sp *mlxsw_sp)
10927 {
10928 	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp1_ipip_ops_arr;
10929 	return mlxsw_sp_ipips_init(mlxsw_sp);
10930 }
10931 
10932 static int mlxsw_sp2_ipips_init(struct mlxsw_sp *mlxsw_sp)
10933 {
10934 	mlxsw_sp->router->ipip_ops_arr = mlxsw_sp2_ipip_ops_arr;
10935 	return mlxsw_sp_ipips_init(mlxsw_sp);
10936 }
10937 
10938 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
10939 {
10940 	WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
10941 }
10942 
10943 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
10944 {
10945 	struct mlxsw_sp_router *router;
10946 
10947 	/* Flush pending FIB notifications and then flush the device's
10948 	 * table before requesting another dump. The FIB notification
10949 	 * block is unregistered, so no need to take RTNL.
10950 	 */
10951 	mlxsw_core_flush_owq();
10952 	router = container_of(nb, struct mlxsw_sp_router, fib_nb);
10953 	mlxsw_sp_router_fib_flush(router->mlxsw_sp);
10954 }
10955 
10956 #ifdef CONFIG_IP_ROUTE_MULTIPATH
10957 struct mlxsw_sp_mp_hash_config {
10958 	DECLARE_BITMAP(headers, __MLXSW_REG_RECR2_HEADER_CNT);
10959 	DECLARE_BITMAP(fields, __MLXSW_REG_RECR2_FIELD_CNT);
10960 	DECLARE_BITMAP(inner_headers, __MLXSW_REG_RECR2_HEADER_CNT);
10961 	DECLARE_BITMAP(inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT);
10962 	bool inc_parsing_depth;
10963 };
10964 
10965 #define MLXSW_SP_MP_HASH_HEADER_SET(_headers, _header) \
10966 	bitmap_set(_headers, MLXSW_REG_RECR2_##_header, 1)
10967 
10968 #define MLXSW_SP_MP_HASH_FIELD_SET(_fields, _field) \
10969 	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, 1)
10970 
10971 #define MLXSW_SP_MP_HASH_FIELD_RANGE_SET(_fields, _field, _nr) \
10972 	bitmap_set(_fields, MLXSW_REG_RECR2_##_field, _nr)
10973 
10974 static void mlxsw_sp_mp_hash_inner_l3(struct mlxsw_sp_mp_hash_config *config)
10975 {
10976 	unsigned long *inner_headers = config->inner_headers;
10977 	unsigned long *inner_fields = config->inner_fields;
10978 
10979 	/* IPv4 inner */
10980 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
10981 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
10982 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
10983 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
10984 	/* IPv6 inner */
10985 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
10986 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
10987 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
10988 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
10989 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
10990 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
10991 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
10992 	MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
10993 }
10994 
10995 static void mlxsw_sp_mp4_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
10996 {
10997 	unsigned long *headers = config->headers;
10998 	unsigned long *fields = config->fields;
10999 
11000 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11001 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11002 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11003 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11004 }
11005 
11006 static void
11007 mlxsw_sp_mp_hash_inner_custom(struct mlxsw_sp_mp_hash_config *config,
11008 			      u32 hash_fields)
11009 {
11010 	unsigned long *inner_headers = config->inner_headers;
11011 	unsigned long *inner_fields = config->inner_fields;
11012 
11013 	/* IPv4 Inner */
11014 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_NOT_TCP_NOT_UDP);
11015 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV4_EN_TCP_UDP);
11016 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
11017 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_SIP0, 4);
11018 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
11019 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV4_DIP0, 4);
11020 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11021 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV4_PROTOCOL);
11022 	/* IPv6 inner */
11023 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_NOT_TCP_NOT_UDP);
11024 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, IPV6_EN_TCP_UDP);
11025 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP) {
11026 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_SIP0_7);
11027 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_SIP8, 8);
11028 	}
11029 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP) {
11030 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_DIP0_7);
11031 		MLXSW_SP_MP_HASH_FIELD_RANGE_SET(inner_fields, INNER_IPV6_DIP8, 8);
11032 	}
11033 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
11034 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_NEXT_HEADER);
11035 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
11036 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_IPV6_FLOW_LABEL);
11037 	/* L4 inner */
11038 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV4);
11039 	MLXSW_SP_MP_HASH_HEADER_SET(inner_headers, TCP_UDP_EN_IPV6);
11040 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
11041 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_SPORT);
11042 	if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
11043 		MLXSW_SP_MP_HASH_FIELD_SET(inner_fields, INNER_TCP_UDP_DPORT);
11044 }
11045 
11046 static void mlxsw_sp_mp4_hash_init(struct mlxsw_sp *mlxsw_sp,
11047 				   struct mlxsw_sp_mp_hash_config *config)
11048 {
11049 	struct net *net = mlxsw_sp_net(mlxsw_sp);
11050 	unsigned long *headers = config->headers;
11051 	unsigned long *fields = config->fields;
11052 	u32 hash_fields;
11053 
11054 	switch (READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_policy)) {
11055 	case 0:
11056 		mlxsw_sp_mp4_hash_outer_addr(config);
11057 		break;
11058 	case 1:
11059 		mlxsw_sp_mp4_hash_outer_addr(config);
11060 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11061 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11062 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11063 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11064 		break;
11065 	case 2:
11066 		/* Outer */
11067 		mlxsw_sp_mp4_hash_outer_addr(config);
11068 		/* Inner */
11069 		mlxsw_sp_mp_hash_inner_l3(config);
11070 		break;
11071 	case 3:
11072 		hash_fields = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_fields);
11073 		/* Outer */
11074 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_NOT_TCP_NOT_UDP);
11075 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV4_EN_TCP_UDP);
11076 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV4);
11077 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
11078 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_SIP0, 4);
11079 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
11080 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV4_DIP0, 4);
11081 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11082 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV4_PROTOCOL);
11083 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11084 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11085 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11086 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11087 		/* Inner */
11088 		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11089 		break;
11090 	}
11091 }
11092 
11093 static void mlxsw_sp_mp6_hash_outer_addr(struct mlxsw_sp_mp_hash_config *config)
11094 {
11095 	unsigned long *headers = config->headers;
11096 	unsigned long *fields = config->fields;
11097 
11098 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11099 	MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11100 	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11101 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11102 	MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11103 	MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11104 }
11105 
11106 static void mlxsw_sp_mp6_hash_init(struct mlxsw_sp *mlxsw_sp,
11107 				   struct mlxsw_sp_mp_hash_config *config)
11108 {
11109 	u32 hash_fields = ip6_multipath_hash_fields(mlxsw_sp_net(mlxsw_sp));
11110 	unsigned long *headers = config->headers;
11111 	unsigned long *fields = config->fields;
11112 
11113 	switch (ip6_multipath_hash_policy(mlxsw_sp_net(mlxsw_sp))) {
11114 	case 0:
11115 		mlxsw_sp_mp6_hash_outer_addr(config);
11116 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11117 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11118 		break;
11119 	case 1:
11120 		mlxsw_sp_mp6_hash_outer_addr(config);
11121 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11122 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11123 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11124 		MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11125 		break;
11126 	case 2:
11127 		/* Outer */
11128 		mlxsw_sp_mp6_hash_outer_addr(config);
11129 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11130 		MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11131 		/* Inner */
11132 		mlxsw_sp_mp_hash_inner_l3(config);
11133 		config->inc_parsing_depth = true;
11134 		break;
11135 	case 3:
11136 		/* Outer */
11137 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_NOT_TCP_NOT_UDP);
11138 		MLXSW_SP_MP_HASH_HEADER_SET(headers, IPV6_EN_TCP_UDP);
11139 		MLXSW_SP_MP_HASH_HEADER_SET(headers, TCP_UDP_EN_IPV6);
11140 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP) {
11141 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_SIP0_7);
11142 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_SIP8, 8);
11143 		}
11144 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP) {
11145 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_DIP0_7);
11146 			MLXSW_SP_MP_HASH_FIELD_RANGE_SET(fields, IPV6_DIP8, 8);
11147 		}
11148 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
11149 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_NEXT_HEADER);
11150 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_FLOWLABEL)
11151 			MLXSW_SP_MP_HASH_FIELD_SET(fields, IPV6_FLOW_LABEL);
11152 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
11153 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_SPORT);
11154 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
11155 			MLXSW_SP_MP_HASH_FIELD_SET(fields, TCP_UDP_DPORT);
11156 		/* Inner */
11157 		mlxsw_sp_mp_hash_inner_custom(config, hash_fields);
11158 		if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK)
11159 			config->inc_parsing_depth = true;
11160 		break;
11161 	}
11162 }
11163 
11164 static int mlxsw_sp_mp_hash_parsing_depth_adjust(struct mlxsw_sp *mlxsw_sp,
11165 						 bool old_inc_parsing_depth,
11166 						 bool new_inc_parsing_depth)
11167 {
11168 	int err;
11169 
11170 	if (!old_inc_parsing_depth && new_inc_parsing_depth) {
11171 		err = mlxsw_sp_parsing_depth_inc(mlxsw_sp);
11172 		if (err)
11173 			return err;
11174 		mlxsw_sp->router->inc_parsing_depth = true;
11175 	} else if (old_inc_parsing_depth && !new_inc_parsing_depth) {
11176 		mlxsw_sp_parsing_depth_dec(mlxsw_sp);
11177 		mlxsw_sp->router->inc_parsing_depth = false;
11178 	}
11179 
11180 	return 0;
11181 }
11182 
11183 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11184 {
11185 	bool old_inc_parsing_depth, new_inc_parsing_depth;
11186 	struct mlxsw_sp_mp_hash_config config = {};
11187 	char recr2_pl[MLXSW_REG_RECR2_LEN];
11188 	unsigned long bit;
11189 	u32 seed;
11190 	int err;
11191 
11192 	seed = jhash(mlxsw_sp->base_mac, sizeof(mlxsw_sp->base_mac), 0);
11193 	mlxsw_reg_recr2_pack(recr2_pl, seed);
11194 	mlxsw_sp_mp4_hash_init(mlxsw_sp, &config);
11195 	mlxsw_sp_mp6_hash_init(mlxsw_sp, &config);
11196 
11197 	old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11198 	new_inc_parsing_depth = config.inc_parsing_depth;
11199 	err = mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp,
11200 						    old_inc_parsing_depth,
11201 						    new_inc_parsing_depth);
11202 	if (err)
11203 		return err;
11204 
11205 	for_each_set_bit(bit, config.headers, __MLXSW_REG_RECR2_HEADER_CNT)
11206 		mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, bit, 1);
11207 	for_each_set_bit(bit, config.fields, __MLXSW_REG_RECR2_FIELD_CNT)
11208 		mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, bit, 1);
11209 	for_each_set_bit(bit, config.inner_headers, __MLXSW_REG_RECR2_HEADER_CNT)
11210 		mlxsw_reg_recr2_inner_header_enables_set(recr2_pl, bit, 1);
11211 	for_each_set_bit(bit, config.inner_fields, __MLXSW_REG_RECR2_INNER_FIELD_CNT)
11212 		mlxsw_reg_recr2_inner_header_fields_enable_set(recr2_pl, bit, 1);
11213 
11214 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
11215 	if (err)
11216 		goto err_reg_write;
11217 
11218 	return 0;
11219 
11220 err_reg_write:
11221 	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, new_inc_parsing_depth,
11222 					      old_inc_parsing_depth);
11223 	return err;
11224 }
11225 
11226 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11227 {
11228 	bool old_inc_parsing_depth = mlxsw_sp->router->inc_parsing_depth;
11229 
11230 	mlxsw_sp_mp_hash_parsing_depth_adjust(mlxsw_sp, old_inc_parsing_depth,
11231 					      false);
11232 }
11233 #else
11234 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
11235 {
11236 	return 0;
11237 }
11238 
11239 static void mlxsw_sp_mp_hash_fini(struct mlxsw_sp *mlxsw_sp)
11240 {
11241 }
11242 #endif
11243 
11244 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
11245 {
11246 	char rdpm_pl[MLXSW_REG_RDPM_LEN];
11247 	unsigned int i;
11248 
11249 	MLXSW_REG_ZERO(rdpm, rdpm_pl);
11250 
11251 	/* HW is determining switch priority based on DSCP-bits, but the
11252 	 * kernel is still doing that based on the ToS. Since there's a
11253 	 * mismatch in bits we need to make sure to translate the right
11254 	 * value ToS would observe, skipping the 2 least-significant ECN bits.
11255 	 */
11256 	for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
11257 		mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
11258 
11259 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
11260 }
11261 
11262 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
11263 {
11264 	struct net *net = mlxsw_sp_net(mlxsw_sp);
11265 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
11266 	u64 max_rifs;
11267 	bool usp;
11268 
11269 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
11270 		return -EIO;
11271 	max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
11272 	usp = READ_ONCE(net->ipv4.sysctl_ip_fwd_update_priority);
11273 
11274 	mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
11275 	mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
11276 	mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
11277 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
11278 }
11279 
11280 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11281 {
11282 	char rgcr_pl[MLXSW_REG_RGCR_LEN];
11283 
11284 	mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
11285 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
11286 }
11287 
11288 static int mlxsw_sp_lb_rif_init(struct mlxsw_sp *mlxsw_sp,
11289 				struct netlink_ext_ack *extack)
11290 {
11291 	struct mlxsw_sp_router *router = mlxsw_sp->router;
11292 	struct mlxsw_sp_rif *lb_rif;
11293 	int err;
11294 
11295 	router->lb_crif = mlxsw_sp_crif_alloc(NULL);
11296 	if (!router->lb_crif)
11297 		return -ENOMEM;
11298 
11299 	/* Create a generic loopback RIF associated with the main table
11300 	 * (default VRF). Any table can be used, but the main table exists
11301 	 * anyway, so we do not waste resources. Loopback RIFs are usually
11302 	 * created with a NULL CRIF, but this RIF is used as a fallback RIF
11303 	 * for blackhole nexthops, and nexthops expect to have a valid CRIF.
11304 	 */
11305 	lb_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, RT_TABLE_MAIN, router->lb_crif,
11306 				     extack);
11307 	if (IS_ERR(lb_rif)) {
11308 		err = PTR_ERR(lb_rif);
11309 		goto err_ul_rif_get;
11310 	}
11311 
11312 	return 0;
11313 
11314 err_ul_rif_get:
11315 	mlxsw_sp_crif_free(router->lb_crif);
11316 	return err;
11317 }
11318 
11319 static void mlxsw_sp_lb_rif_fini(struct mlxsw_sp *mlxsw_sp)
11320 {
11321 	mlxsw_sp_ul_rif_put(mlxsw_sp->router->lb_crif->rif);
11322 	mlxsw_sp_crif_free(mlxsw_sp->router->lb_crif);
11323 }
11324 
11325 static int mlxsw_sp1_router_init(struct mlxsw_sp *mlxsw_sp)
11326 {
11327 	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp1_adj_grp_size_ranges);
11328 
11329 	mlxsw_sp->router->rif_ops_arr = mlxsw_sp1_rif_ops_arr;
11330 	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp1_adj_grp_size_ranges;
11331 	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11332 
11333 	return 0;
11334 }
11335 
11336 const struct mlxsw_sp_router_ops mlxsw_sp1_router_ops = {
11337 	.init = mlxsw_sp1_router_init,
11338 	.ipips_init = mlxsw_sp1_ipips_init,
11339 };
11340 
11341 static int mlxsw_sp2_router_init(struct mlxsw_sp *mlxsw_sp)
11342 {
11343 	size_t size_ranges_count = ARRAY_SIZE(mlxsw_sp2_adj_grp_size_ranges);
11344 
11345 	mlxsw_sp->router->rif_ops_arr = mlxsw_sp2_rif_ops_arr;
11346 	mlxsw_sp->router->adj_grp_size_ranges = mlxsw_sp2_adj_grp_size_ranges;
11347 	mlxsw_sp->router->adj_grp_size_ranges_count = size_ranges_count;
11348 
11349 	return 0;
11350 }
11351 
11352 const struct mlxsw_sp_router_ops mlxsw_sp2_router_ops = {
11353 	.init = mlxsw_sp2_router_init,
11354 	.ipips_init = mlxsw_sp2_ipips_init,
11355 };
11356 
11357 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp,
11358 			 struct netlink_ext_ack *extack)
11359 {
11360 	struct mlxsw_sp_router *router;
11361 	struct notifier_block *nb;
11362 	int err;
11363 
11364 	router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
11365 	if (!router)
11366 		return -ENOMEM;
11367 	mutex_init(&router->lock);
11368 	mlxsw_sp->router = router;
11369 	router->mlxsw_sp = mlxsw_sp;
11370 
11371 	err = mlxsw_sp->router_ops->init(mlxsw_sp);
11372 	if (err)
11373 		goto err_router_ops_init;
11374 
11375 	INIT_LIST_HEAD(&mlxsw_sp->router->nh_res_grp_list);
11376 	INIT_DELAYED_WORK(&mlxsw_sp->router->nh_grp_activity_dw,
11377 			  mlxsw_sp_nh_grp_activity_work);
11378 	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
11379 	err = __mlxsw_sp_router_init(mlxsw_sp);
11380 	if (err)
11381 		goto err_router_init;
11382 
11383 	err = mlxsw_sp->router_ops->ipips_init(mlxsw_sp);
11384 	if (err)
11385 		goto err_ipips_init;
11386 
11387 	err = rhashtable_init(&mlxsw_sp->router->crif_ht,
11388 			      &mlxsw_sp_crif_ht_params);
11389 	if (err)
11390 		goto err_crif_ht_init;
11391 
11392 	err = mlxsw_sp_rifs_init(mlxsw_sp);
11393 	if (err)
11394 		goto err_rifs_init;
11395 
11396 	err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
11397 			      &mlxsw_sp_nexthop_ht_params);
11398 	if (err)
11399 		goto err_nexthop_ht_init;
11400 
11401 	err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
11402 			      &mlxsw_sp_nexthop_group_ht_params);
11403 	if (err)
11404 		goto err_nexthop_group_ht_init;
11405 
11406 	INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
11407 	err = mlxsw_sp_lpm_init(mlxsw_sp);
11408 	if (err)
11409 		goto err_lpm_init;
11410 
11411 	err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
11412 	if (err)
11413 		goto err_mr_init;
11414 
11415 	err = mlxsw_sp_vrs_init(mlxsw_sp);
11416 	if (err)
11417 		goto err_vrs_init;
11418 
11419 	err = mlxsw_sp_lb_rif_init(mlxsw_sp, extack);
11420 	if (err)
11421 		goto err_lb_rif_init;
11422 
11423 	err = mlxsw_sp_neigh_init(mlxsw_sp);
11424 	if (err)
11425 		goto err_neigh_init;
11426 
11427 	err = mlxsw_sp_mp_hash_init(mlxsw_sp);
11428 	if (err)
11429 		goto err_mp_hash_init;
11430 
11431 	err = mlxsw_sp_dscp_init(mlxsw_sp);
11432 	if (err)
11433 		goto err_dscp_init;
11434 
11435 	router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
11436 	err = register_inetaddr_notifier(&router->inetaddr_nb);
11437 	if (err)
11438 		goto err_register_inetaddr_notifier;
11439 
11440 	router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
11441 	err = register_inet6addr_notifier(&router->inet6addr_nb);
11442 	if (err)
11443 		goto err_register_inet6addr_notifier;
11444 
11445 	router->inetaddr_valid_nb.notifier_call = mlxsw_sp_inetaddr_valid_event;
11446 	err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11447 	if (err)
11448 		goto err_register_inetaddr_valid_notifier;
11449 
11450 	nb = &router->inet6addr_valid_nb;
11451 	nb->notifier_call = mlxsw_sp_inet6addr_valid_event;
11452 	err = register_inet6addr_validator_notifier(nb);
11453 	if (err)
11454 		goto err_register_inet6addr_valid_notifier;
11455 
11456 	mlxsw_sp->router->netevent_nb.notifier_call =
11457 		mlxsw_sp_router_netevent_event;
11458 	err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
11459 	if (err)
11460 		goto err_register_netevent_notifier;
11461 
11462 	mlxsw_sp->router->netdevice_nb.notifier_call =
11463 		mlxsw_sp_router_netdevice_event;
11464 	err = register_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
11465 					      &mlxsw_sp->router->netdevice_nb);
11466 	if (err)
11467 		goto err_register_netdev_notifier;
11468 
11469 	mlxsw_sp->router->nexthop_nb.notifier_call =
11470 		mlxsw_sp_nexthop_obj_event;
11471 	err = register_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11472 					&mlxsw_sp->router->nexthop_nb,
11473 					extack);
11474 	if (err)
11475 		goto err_register_nexthop_notifier;
11476 
11477 	mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
11478 	err = register_fib_notifier(mlxsw_sp_net(mlxsw_sp),
11479 				    &mlxsw_sp->router->fib_nb,
11480 				    mlxsw_sp_router_fib_dump_flush, extack);
11481 	if (err)
11482 		goto err_register_fib_notifier;
11483 
11484 	return 0;
11485 
11486 err_register_fib_notifier:
11487 	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11488 				    &mlxsw_sp->router->nexthop_nb);
11489 err_register_nexthop_notifier:
11490 	unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
11491 					  &router->netdevice_nb);
11492 err_register_netdev_notifier:
11493 	unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
11494 err_register_netevent_notifier:
11495 	unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb);
11496 err_register_inet6addr_valid_notifier:
11497 	unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11498 err_register_inetaddr_valid_notifier:
11499 	unregister_inet6addr_notifier(&router->inet6addr_nb);
11500 err_register_inet6addr_notifier:
11501 	unregister_inetaddr_notifier(&router->inetaddr_nb);
11502 err_register_inetaddr_notifier:
11503 	mlxsw_core_flush_owq();
11504 err_dscp_init:
11505 	mlxsw_sp_mp_hash_fini(mlxsw_sp);
11506 err_mp_hash_init:
11507 	mlxsw_sp_neigh_fini(mlxsw_sp);
11508 err_neigh_init:
11509 	mlxsw_sp_lb_rif_fini(mlxsw_sp);
11510 err_lb_rif_init:
11511 	mlxsw_sp_vrs_fini(mlxsw_sp);
11512 err_vrs_init:
11513 	mlxsw_sp_mr_fini(mlxsw_sp);
11514 err_mr_init:
11515 	mlxsw_sp_lpm_fini(mlxsw_sp);
11516 err_lpm_init:
11517 	rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
11518 err_nexthop_group_ht_init:
11519 	rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
11520 err_nexthop_ht_init:
11521 	mlxsw_sp_rifs_fini(mlxsw_sp);
11522 err_rifs_init:
11523 	rhashtable_destroy(&mlxsw_sp->router->crif_ht);
11524 err_crif_ht_init:
11525 	mlxsw_sp_ipips_fini(mlxsw_sp);
11526 err_ipips_init:
11527 	__mlxsw_sp_router_fini(mlxsw_sp);
11528 err_router_init:
11529 	cancel_delayed_work_sync(&mlxsw_sp->router->nh_grp_activity_dw);
11530 err_router_ops_init:
11531 	mutex_destroy(&mlxsw_sp->router->lock);
11532 	kfree(mlxsw_sp->router);
11533 	return err;
11534 }
11535 
11536 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
11537 {
11538 	struct mlxsw_sp_router *router = mlxsw_sp->router;
11539 
11540 	unregister_fib_notifier(mlxsw_sp_net(mlxsw_sp), &router->fib_nb);
11541 	unregister_nexthop_notifier(mlxsw_sp_net(mlxsw_sp),
11542 				    &router->nexthop_nb);
11543 	unregister_netdevice_notifier_net(mlxsw_sp_net(mlxsw_sp),
11544 					  &router->netdevice_nb);
11545 	unregister_netevent_notifier(&router->netevent_nb);
11546 	unregister_inet6addr_validator_notifier(&router->inet6addr_valid_nb);
11547 	unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
11548 	unregister_inet6addr_notifier(&router->inet6addr_nb);
11549 	unregister_inetaddr_notifier(&router->inetaddr_nb);
11550 	mlxsw_core_flush_owq();
11551 	mlxsw_sp_mp_hash_fini(mlxsw_sp);
11552 	mlxsw_sp_neigh_fini(mlxsw_sp);
11553 	mlxsw_sp_lb_rif_fini(mlxsw_sp);
11554 	mlxsw_sp_vrs_fini(mlxsw_sp);
11555 	mlxsw_sp_mr_fini(mlxsw_sp);
11556 	mlxsw_sp_lpm_fini(mlxsw_sp);
11557 	rhashtable_destroy(&router->nexthop_group_ht);
11558 	rhashtable_destroy(&router->nexthop_ht);
11559 	mlxsw_sp_rifs_fini(mlxsw_sp);
11560 	rhashtable_destroy(&mlxsw_sp->router->crif_ht);
11561 	mlxsw_sp_ipips_fini(mlxsw_sp);
11562 	__mlxsw_sp_router_fini(mlxsw_sp);
11563 	cancel_delayed_work_sync(&router->nh_grp_activity_dw);
11564 	mutex_destroy(&router->lock);
11565 	kfree(router);
11566 }
11567