1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <net/devlink.h>
6 
7 #include "spectrum.h"
8 #include "spectrum_dpipe.h"
9 #include "spectrum_router.h"
10 
11 enum mlxsw_sp_field_metadata_id {
12 	MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
13 	MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
14 	MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
15 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
16 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
17 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
18 };
19 
20 static struct devlink_dpipe_field mlxsw_sp_dpipe_fields_metadata[] = {
21 	{
22 		.name = "erif_port",
23 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
24 		.bitwidth = 32,
25 		.mapping_type = DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX,
26 	},
27 	{
28 		.name = "l3_forward",
29 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
30 		.bitwidth = 1,
31 	},
32 	{
33 		.name = "l3_drop",
34 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
35 		.bitwidth = 1,
36 	},
37 	{
38 		.name = "adj_index",
39 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
40 		.bitwidth = 32,
41 	},
42 	{
43 		.name = "adj_size",
44 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
45 		.bitwidth = 32,
46 	},
47 	{
48 		.name = "adj_hash_index",
49 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
50 		.bitwidth = 32,
51 	},
52 };
53 
54 enum mlxsw_sp_dpipe_header_id {
55 	MLXSW_SP_DPIPE_HEADER_METADATA,
56 };
57 
58 static struct devlink_dpipe_header mlxsw_sp_dpipe_header_metadata = {
59 	.name = "mlxsw_meta",
60 	.id = MLXSW_SP_DPIPE_HEADER_METADATA,
61 	.fields = mlxsw_sp_dpipe_fields_metadata,
62 	.fields_count = ARRAY_SIZE(mlxsw_sp_dpipe_fields_metadata),
63 };
64 
65 static struct devlink_dpipe_header *mlxsw_dpipe_headers[] = {
66 	&mlxsw_sp_dpipe_header_metadata,
67 	&devlink_dpipe_header_ethernet,
68 	&devlink_dpipe_header_ipv4,
69 	&devlink_dpipe_header_ipv6,
70 };
71 
72 static struct devlink_dpipe_headers mlxsw_sp_dpipe_headers = {
73 	.headers = mlxsw_dpipe_headers,
74 	.headers_count = ARRAY_SIZE(mlxsw_dpipe_headers),
75 };
76 
77 static int mlxsw_sp_dpipe_table_erif_actions_dump(void *priv,
78 						  struct sk_buff *skb)
79 {
80 	struct devlink_dpipe_action action = {0};
81 	int err;
82 
83 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
84 	action.header = &mlxsw_sp_dpipe_header_metadata;
85 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
86 
87 	err = devlink_dpipe_action_put(skb, &action);
88 	if (err)
89 		return err;
90 
91 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
92 	action.header = &mlxsw_sp_dpipe_header_metadata;
93 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP;
94 
95 	return devlink_dpipe_action_put(skb, &action);
96 }
97 
98 static int mlxsw_sp_dpipe_table_erif_matches_dump(void *priv,
99 						  struct sk_buff *skb)
100 {
101 	struct devlink_dpipe_match match = {0};
102 
103 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
104 	match.header = &mlxsw_sp_dpipe_header_metadata;
105 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
106 
107 	return devlink_dpipe_match_put(skb, &match);
108 }
109 
110 static void
111 mlxsw_sp_erif_match_action_prepare(struct devlink_dpipe_match *match,
112 				   struct devlink_dpipe_action *action)
113 {
114 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
115 	action->header = &mlxsw_sp_dpipe_header_metadata;
116 	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
117 
118 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
119 	match->header = &mlxsw_sp_dpipe_header_metadata;
120 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
121 }
122 
123 static int mlxsw_sp_erif_entry_prepare(struct devlink_dpipe_entry *entry,
124 				       struct devlink_dpipe_value *match_value,
125 				       struct devlink_dpipe_match *match,
126 				       struct devlink_dpipe_value *action_value,
127 				       struct devlink_dpipe_action *action)
128 {
129 	entry->match_values = match_value;
130 	entry->match_values_count = 1;
131 
132 	entry->action_values = action_value;
133 	entry->action_values_count = 1;
134 
135 	match_value->match = match;
136 	match_value->value_size = sizeof(u32);
137 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
138 	if (!match_value->value)
139 		return -ENOMEM;
140 
141 	action_value->action = action;
142 	action_value->value_size = sizeof(u32);
143 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
144 	if (!action_value->value)
145 		goto err_action_alloc;
146 	return 0;
147 
148 err_action_alloc:
149 	kfree(match_value->value);
150 	return -ENOMEM;
151 }
152 
153 static int mlxsw_sp_erif_entry_get(struct mlxsw_sp *mlxsw_sp,
154 				   struct devlink_dpipe_entry *entry,
155 				   struct mlxsw_sp_rif *rif,
156 				   bool counters_enabled)
157 {
158 	u32 *action_value;
159 	u32 *rif_value;
160 	u64 cnt;
161 	int err;
162 
163 	/* Set Match RIF index */
164 	rif_value = entry->match_values->value;
165 	*rif_value = mlxsw_sp_rif_index(rif);
166 	entry->match_values->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
167 	entry->match_values->mapping_valid = true;
168 
169 	/* Set Action Forwarding */
170 	action_value = entry->action_values->value;
171 	*action_value = 1;
172 
173 	entry->counter_valid = false;
174 	entry->counter = 0;
175 	entry->index = mlxsw_sp_rif_index(rif);
176 
177 	if (!counters_enabled)
178 		return 0;
179 
180 	err = mlxsw_sp_rif_counter_value_get(mlxsw_sp, rif,
181 					     MLXSW_SP_RIF_COUNTER_EGRESS,
182 					     &cnt);
183 	if (!err) {
184 		entry->counter = cnt;
185 		entry->counter_valid = true;
186 	}
187 	return 0;
188 }
189 
190 static int
191 mlxsw_sp_dpipe_table_erif_entries_dump(void *priv, bool counters_enabled,
192 				       struct devlink_dpipe_dump_ctx *dump_ctx)
193 {
194 	struct devlink_dpipe_value match_value, action_value;
195 	struct devlink_dpipe_action action = {0};
196 	struct devlink_dpipe_match match = {0};
197 	struct devlink_dpipe_entry entry = {0};
198 	struct mlxsw_sp *mlxsw_sp = priv;
199 	unsigned int rif_count;
200 	int i, j;
201 	int err;
202 
203 	memset(&match_value, 0, sizeof(match_value));
204 	memset(&action_value, 0, sizeof(action_value));
205 
206 	mlxsw_sp_erif_match_action_prepare(&match, &action);
207 	err = mlxsw_sp_erif_entry_prepare(&entry, &match_value, &match,
208 					  &action_value, &action);
209 	if (err)
210 		return err;
211 
212 	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
213 	rtnl_lock();
214 	i = 0;
215 start_again:
216 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
217 	if (err)
218 		goto err_ctx_prepare;
219 	j = 0;
220 	for (; i < rif_count; i++) {
221 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
222 
223 		if (!rif || !mlxsw_sp_rif_dev(rif))
224 			continue;
225 		err = mlxsw_sp_erif_entry_get(mlxsw_sp, &entry, rif,
226 					      counters_enabled);
227 		if (err)
228 			goto err_entry_get;
229 		err = devlink_dpipe_entry_ctx_append(dump_ctx, &entry);
230 		if (err) {
231 			if (err == -EMSGSIZE) {
232 				if (!j)
233 					goto err_entry_append;
234 				break;
235 			}
236 			goto err_entry_append;
237 		}
238 		j++;
239 	}
240 
241 	devlink_dpipe_entry_ctx_close(dump_ctx);
242 	if (i != rif_count)
243 		goto start_again;
244 	rtnl_unlock();
245 
246 	devlink_dpipe_entry_clear(&entry);
247 	return 0;
248 err_entry_append:
249 err_entry_get:
250 err_ctx_prepare:
251 	rtnl_unlock();
252 	devlink_dpipe_entry_clear(&entry);
253 	return err;
254 }
255 
256 static int mlxsw_sp_dpipe_table_erif_counters_update(void *priv, bool enable)
257 {
258 	struct mlxsw_sp *mlxsw_sp = priv;
259 	int i;
260 
261 	rtnl_lock();
262 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
263 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
264 
265 		if (!rif)
266 			continue;
267 		if (enable)
268 			mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif,
269 						   MLXSW_SP_RIF_COUNTER_EGRESS);
270 		else
271 			mlxsw_sp_rif_counter_free(mlxsw_sp, rif,
272 						  MLXSW_SP_RIF_COUNTER_EGRESS);
273 	}
274 	rtnl_unlock();
275 	return 0;
276 }
277 
278 static u64 mlxsw_sp_dpipe_table_erif_size_get(void *priv)
279 {
280 	struct mlxsw_sp *mlxsw_sp = priv;
281 
282 	return MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
283 }
284 
285 static struct devlink_dpipe_table_ops mlxsw_sp_erif_ops = {
286 	.matches_dump = mlxsw_sp_dpipe_table_erif_matches_dump,
287 	.actions_dump = mlxsw_sp_dpipe_table_erif_actions_dump,
288 	.entries_dump = mlxsw_sp_dpipe_table_erif_entries_dump,
289 	.counters_set_update = mlxsw_sp_dpipe_table_erif_counters_update,
290 	.size_get = mlxsw_sp_dpipe_table_erif_size_get,
291 };
292 
293 static int mlxsw_sp_dpipe_erif_table_init(struct mlxsw_sp *mlxsw_sp)
294 {
295 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
296 
297 	return devlink_dpipe_table_register(devlink,
298 					    MLXSW_SP_DPIPE_TABLE_NAME_ERIF,
299 					    &mlxsw_sp_erif_ops,
300 					    mlxsw_sp, false);
301 }
302 
303 static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp)
304 {
305 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
306 
307 	devlink_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF);
308 }
309 
310 static int mlxsw_sp_dpipe_table_host_matches_dump(struct sk_buff *skb, int type)
311 {
312 	struct devlink_dpipe_match match = {0};
313 	int err;
314 
315 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
316 	match.header = &mlxsw_sp_dpipe_header_metadata;
317 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
318 
319 	err = devlink_dpipe_match_put(skb, &match);
320 	if (err)
321 		return err;
322 
323 	switch (type) {
324 	case AF_INET:
325 		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
326 		match.header = &devlink_dpipe_header_ipv4;
327 		match.field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
328 		break;
329 	case AF_INET6:
330 		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
331 		match.header = &devlink_dpipe_header_ipv6;
332 		match.field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
333 		break;
334 	default:
335 		WARN_ON(1);
336 		return -EINVAL;
337 	}
338 
339 	return devlink_dpipe_match_put(skb, &match);
340 }
341 
342 static int
343 mlxsw_sp_dpipe_table_host4_matches_dump(void *priv, struct sk_buff *skb)
344 {
345 	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET);
346 }
347 
348 static int
349 mlxsw_sp_dpipe_table_host_actions_dump(void *priv, struct sk_buff *skb)
350 {
351 	struct devlink_dpipe_action action = {0};
352 
353 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
354 	action.header = &devlink_dpipe_header_ethernet;
355 	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
356 
357 	return devlink_dpipe_action_put(skb, &action);
358 }
359 
360 enum mlxsw_sp_dpipe_table_host_match {
361 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF,
362 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP,
363 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT,
364 };
365 
366 static void
367 mlxsw_sp_dpipe_table_host_match_action_prepare(struct devlink_dpipe_match *matches,
368 					       struct devlink_dpipe_action *action,
369 					       int type)
370 {
371 	struct devlink_dpipe_match *match;
372 
373 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
374 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
375 	match->header = &mlxsw_sp_dpipe_header_metadata;
376 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
377 
378 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
379 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
380 	switch (type) {
381 	case AF_INET:
382 		match->header = &devlink_dpipe_header_ipv4;
383 		match->field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
384 		break;
385 	case AF_INET6:
386 		match->header = &devlink_dpipe_header_ipv6;
387 		match->field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
388 		break;
389 	default:
390 		WARN_ON(1);
391 		return;
392 	}
393 
394 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
395 	action->header = &devlink_dpipe_header_ethernet;
396 	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
397 }
398 
399 static int
400 mlxsw_sp_dpipe_table_host_entry_prepare(struct devlink_dpipe_entry *entry,
401 					struct devlink_dpipe_value *match_values,
402 					struct devlink_dpipe_match *matches,
403 					struct devlink_dpipe_value *action_value,
404 					struct devlink_dpipe_action *action,
405 					int type)
406 {
407 	struct devlink_dpipe_value *match_value;
408 	struct devlink_dpipe_match *match;
409 
410 	entry->match_values = match_values;
411 	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT;
412 
413 	entry->action_values = action_value;
414 	entry->action_values_count = 1;
415 
416 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
417 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
418 
419 	match_value->match = match;
420 	match_value->value_size = sizeof(u32);
421 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
422 	if (!match_value->value)
423 		return -ENOMEM;
424 
425 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
426 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
427 
428 	match_value->match = match;
429 	switch (type) {
430 	case AF_INET:
431 		match_value->value_size = sizeof(u32);
432 		break;
433 	case AF_INET6:
434 		match_value->value_size = sizeof(struct in6_addr);
435 		break;
436 	default:
437 		WARN_ON(1);
438 		return -EINVAL;
439 	}
440 
441 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
442 	if (!match_value->value)
443 		return -ENOMEM;
444 
445 	action_value->action = action;
446 	action_value->value_size = sizeof(u64);
447 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
448 	if (!action_value->value)
449 		return -ENOMEM;
450 
451 	return 0;
452 }
453 
454 static void
455 __mlxsw_sp_dpipe_table_host_entry_fill(struct devlink_dpipe_entry *entry,
456 				       struct mlxsw_sp_rif *rif,
457 				       unsigned char *ha, void *dip)
458 {
459 	struct devlink_dpipe_value *value;
460 	u32 *rif_value;
461 	u8 *ha_value;
462 
463 	/* Set Match RIF index */
464 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
465 
466 	rif_value = value->value;
467 	*rif_value = mlxsw_sp_rif_index(rif);
468 	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
469 	value->mapping_valid = true;
470 
471 	/* Set Match DIP */
472 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
473 	memcpy(value->value, dip, value->value_size);
474 
475 	/* Set Action DMAC */
476 	value = entry->action_values;
477 	ha_value = value->value;
478 	ether_addr_copy(ha_value, ha);
479 }
480 
481 static void
482 mlxsw_sp_dpipe_table_host4_entry_fill(struct devlink_dpipe_entry *entry,
483 				      struct mlxsw_sp_neigh_entry *neigh_entry,
484 				      struct mlxsw_sp_rif *rif)
485 {
486 	unsigned char *ha;
487 	u32 dip;
488 
489 	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
490 	dip = mlxsw_sp_neigh4_entry_dip(neigh_entry);
491 	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, &dip);
492 }
493 
494 static void
495 mlxsw_sp_dpipe_table_host6_entry_fill(struct devlink_dpipe_entry *entry,
496 				      struct mlxsw_sp_neigh_entry *neigh_entry,
497 				      struct mlxsw_sp_rif *rif)
498 {
499 	struct in6_addr *dip;
500 	unsigned char *ha;
501 
502 	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
503 	dip = mlxsw_sp_neigh6_entry_dip(neigh_entry);
504 
505 	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, dip);
506 }
507 
508 static void
509 mlxsw_sp_dpipe_table_host_entry_fill(struct mlxsw_sp *mlxsw_sp,
510 				     struct devlink_dpipe_entry *entry,
511 				     struct mlxsw_sp_neigh_entry *neigh_entry,
512 				     struct mlxsw_sp_rif *rif,
513 				     int type)
514 {
515 	int err;
516 
517 	switch (type) {
518 	case AF_INET:
519 		mlxsw_sp_dpipe_table_host4_entry_fill(entry, neigh_entry, rif);
520 		break;
521 	case AF_INET6:
522 		mlxsw_sp_dpipe_table_host6_entry_fill(entry, neigh_entry, rif);
523 		break;
524 	default:
525 		WARN_ON(1);
526 		return;
527 	}
528 
529 	err = mlxsw_sp_neigh_counter_get(mlxsw_sp, neigh_entry,
530 					 &entry->counter);
531 	if (!err)
532 		entry->counter_valid = true;
533 }
534 
535 static int
536 mlxsw_sp_dpipe_table_host_entries_get(struct mlxsw_sp *mlxsw_sp,
537 				      struct devlink_dpipe_entry *entry,
538 				      bool counters_enabled,
539 				      struct devlink_dpipe_dump_ctx *dump_ctx,
540 				      int type)
541 {
542 	int rif_neigh_count = 0;
543 	int rif_neigh_skip = 0;
544 	int neigh_count = 0;
545 	int rif_count;
546 	int i, j;
547 	int err;
548 
549 	rtnl_lock();
550 	i = 0;
551 	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
552 start_again:
553 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
554 	if (err)
555 		goto err_ctx_prepare;
556 	j = 0;
557 	rif_neigh_skip = rif_neigh_count;
558 	for (; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
559 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
560 		struct mlxsw_sp_neigh_entry *neigh_entry;
561 
562 		if (!rif)
563 			continue;
564 
565 		rif_neigh_count = 0;
566 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
567 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
568 
569 			if (neigh_type != type)
570 				continue;
571 
572 			if (neigh_type == AF_INET6 &&
573 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
574 				continue;
575 
576 			if (rif_neigh_count < rif_neigh_skip)
577 				goto skip;
578 
579 			mlxsw_sp_dpipe_table_host_entry_fill(mlxsw_sp, entry,
580 							     neigh_entry, rif,
581 							     type);
582 			entry->index = neigh_count;
583 			err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
584 			if (err) {
585 				if (err == -EMSGSIZE) {
586 					if (!j)
587 						goto err_entry_append;
588 					else
589 						goto out;
590 				}
591 				goto err_entry_append;
592 			}
593 			neigh_count++;
594 			j++;
595 skip:
596 			rif_neigh_count++;
597 		}
598 		rif_neigh_skip = 0;
599 	}
600 out:
601 	devlink_dpipe_entry_ctx_close(dump_ctx);
602 	if (i != rif_count)
603 		goto start_again;
604 
605 	rtnl_unlock();
606 	return 0;
607 
608 err_ctx_prepare:
609 err_entry_append:
610 	rtnl_unlock();
611 	return err;
612 }
613 
614 static int
615 mlxsw_sp_dpipe_table_host_entries_dump(struct mlxsw_sp *mlxsw_sp,
616 				       bool counters_enabled,
617 				       struct devlink_dpipe_dump_ctx *dump_ctx,
618 				       int type)
619 {
620 	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
621 	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
622 	struct devlink_dpipe_value action_value;
623 	struct devlink_dpipe_action action = {0};
624 	struct devlink_dpipe_entry entry = {0};
625 	int err;
626 
627 	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
628 			   sizeof(matches[0]));
629 	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
630 				sizeof(match_values[0]));
631 	memset(&action_value, 0, sizeof(action_value));
632 
633 	mlxsw_sp_dpipe_table_host_match_action_prepare(matches, &action, type);
634 	err = mlxsw_sp_dpipe_table_host_entry_prepare(&entry, match_values,
635 						      matches, &action_value,
636 						      &action, type);
637 	if (err)
638 		goto out;
639 
640 	err = mlxsw_sp_dpipe_table_host_entries_get(mlxsw_sp, &entry,
641 						    counters_enabled, dump_ctx,
642 						    type);
643 out:
644 	devlink_dpipe_entry_clear(&entry);
645 	return err;
646 }
647 
648 static int
649 mlxsw_sp_dpipe_table_host4_entries_dump(void *priv, bool counters_enabled,
650 					struct devlink_dpipe_dump_ctx *dump_ctx)
651 {
652 	struct mlxsw_sp *mlxsw_sp = priv;
653 
654 	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
655 						      counters_enabled,
656 						      dump_ctx, AF_INET);
657 }
658 
659 static void
660 mlxsw_sp_dpipe_table_host_counters_update(struct mlxsw_sp *mlxsw_sp,
661 					  bool enable, int type)
662 {
663 	int i;
664 
665 	rtnl_lock();
666 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
667 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
668 		struct mlxsw_sp_neigh_entry *neigh_entry;
669 
670 		if (!rif)
671 			continue;
672 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
673 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
674 
675 			if (neigh_type != type)
676 				continue;
677 
678 			if (neigh_type == AF_INET6 &&
679 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
680 				continue;
681 
682 			mlxsw_sp_neigh_entry_counter_update(mlxsw_sp,
683 							    neigh_entry,
684 							    enable);
685 		}
686 	}
687 	rtnl_unlock();
688 }
689 
690 static int mlxsw_sp_dpipe_table_host4_counters_update(void *priv, bool enable)
691 {
692 	struct mlxsw_sp *mlxsw_sp = priv;
693 
694 	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET);
695 	return 0;
696 }
697 
698 static u64
699 mlxsw_sp_dpipe_table_host_size_get(struct mlxsw_sp *mlxsw_sp, int type)
700 {
701 	u64 size = 0;
702 	int i;
703 
704 	rtnl_lock();
705 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
706 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
707 		struct mlxsw_sp_neigh_entry *neigh_entry;
708 
709 		if (!rif)
710 			continue;
711 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
712 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
713 
714 			if (neigh_type != type)
715 				continue;
716 
717 			if (neigh_type == AF_INET6 &&
718 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
719 				continue;
720 
721 			size++;
722 		}
723 	}
724 	rtnl_unlock();
725 
726 	return size;
727 }
728 
729 static u64 mlxsw_sp_dpipe_table_host4_size_get(void *priv)
730 {
731 	struct mlxsw_sp *mlxsw_sp = priv;
732 
733 	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET);
734 }
735 
736 static struct devlink_dpipe_table_ops mlxsw_sp_host4_ops = {
737 	.matches_dump = mlxsw_sp_dpipe_table_host4_matches_dump,
738 	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
739 	.entries_dump = mlxsw_sp_dpipe_table_host4_entries_dump,
740 	.counters_set_update = mlxsw_sp_dpipe_table_host4_counters_update,
741 	.size_get = mlxsw_sp_dpipe_table_host4_size_get,
742 };
743 
744 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4 1
745 
746 static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp)
747 {
748 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
749 	int err;
750 
751 	err = devlink_dpipe_table_register(devlink,
752 					   MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
753 					   &mlxsw_sp_host4_ops,
754 					   mlxsw_sp, false);
755 	if (err)
756 		return err;
757 
758 	err = devlink_dpipe_table_resource_set(devlink,
759 					       MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
760 					       MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
761 					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4);
762 	if (err)
763 		goto err_resource_set;
764 
765 	return 0;
766 
767 err_resource_set:
768 	devlink_dpipe_table_unregister(devlink,
769 				       MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
770 	return err;
771 }
772 
773 static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp)
774 {
775 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
776 
777 	devlink_dpipe_table_unregister(devlink,
778 				       MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
779 }
780 
781 static int
782 mlxsw_sp_dpipe_table_host6_matches_dump(void *priv, struct sk_buff *skb)
783 {
784 	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET6);
785 }
786 
787 static int
788 mlxsw_sp_dpipe_table_host6_entries_dump(void *priv, bool counters_enabled,
789 					struct devlink_dpipe_dump_ctx *dump_ctx)
790 {
791 	struct mlxsw_sp *mlxsw_sp = priv;
792 
793 	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
794 						      counters_enabled,
795 						      dump_ctx, AF_INET6);
796 }
797 
798 static int mlxsw_sp_dpipe_table_host6_counters_update(void *priv, bool enable)
799 {
800 	struct mlxsw_sp *mlxsw_sp = priv;
801 
802 	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET6);
803 	return 0;
804 }
805 
806 static u64 mlxsw_sp_dpipe_table_host6_size_get(void *priv)
807 {
808 	struct mlxsw_sp *mlxsw_sp = priv;
809 
810 	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET6);
811 }
812 
813 static struct devlink_dpipe_table_ops mlxsw_sp_host6_ops = {
814 	.matches_dump = mlxsw_sp_dpipe_table_host6_matches_dump,
815 	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
816 	.entries_dump = mlxsw_sp_dpipe_table_host6_entries_dump,
817 	.counters_set_update = mlxsw_sp_dpipe_table_host6_counters_update,
818 	.size_get = mlxsw_sp_dpipe_table_host6_size_get,
819 };
820 
821 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6 2
822 
823 static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp)
824 {
825 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
826 	int err;
827 
828 	err = devlink_dpipe_table_register(devlink,
829 					   MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
830 					   &mlxsw_sp_host6_ops,
831 					   mlxsw_sp, false);
832 	if (err)
833 		return err;
834 
835 	err = devlink_dpipe_table_resource_set(devlink,
836 					       MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
837 					       MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
838 					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6);
839 	if (err)
840 		goto err_resource_set;
841 
842 	return 0;
843 
844 err_resource_set:
845 	devlink_dpipe_table_unregister(devlink,
846 				       MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
847 	return err;
848 }
849 
850 static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp)
851 {
852 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
853 
854 	devlink_dpipe_table_unregister(devlink,
855 				       MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
856 }
857 
858 static int mlxsw_sp_dpipe_table_adj_matches_dump(void *priv,
859 						 struct sk_buff *skb)
860 {
861 	struct devlink_dpipe_match match = {0};
862 	int err;
863 
864 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
865 	match.header = &mlxsw_sp_dpipe_header_metadata;
866 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
867 
868 	err = devlink_dpipe_match_put(skb, &match);
869 	if (err)
870 		return err;
871 
872 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
873 	match.header = &mlxsw_sp_dpipe_header_metadata;
874 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
875 
876 	err = devlink_dpipe_match_put(skb, &match);
877 	if (err)
878 		return err;
879 
880 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
881 	match.header = &mlxsw_sp_dpipe_header_metadata;
882 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
883 
884 	return devlink_dpipe_match_put(skb, &match);
885 }
886 
887 static int mlxsw_sp_dpipe_table_adj_actions_dump(void *priv,
888 						 struct sk_buff *skb)
889 {
890 	struct devlink_dpipe_action action = {0};
891 	int err;
892 
893 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
894 	action.header = &devlink_dpipe_header_ethernet;
895 	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
896 
897 	err = devlink_dpipe_action_put(skb, &action);
898 	if (err)
899 		return err;
900 
901 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
902 	action.header = &mlxsw_sp_dpipe_header_metadata;
903 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
904 
905 	return devlink_dpipe_action_put(skb, &action);
906 }
907 
908 static u64 mlxsw_sp_dpipe_table_adj_size(struct mlxsw_sp *mlxsw_sp)
909 {
910 	struct mlxsw_sp_nexthop *nh;
911 	u64 size = 0;
912 
913 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router)
914 		if (mlxsw_sp_nexthop_offload(nh) &&
915 		    !mlxsw_sp_nexthop_group_has_ipip(nh))
916 			size++;
917 	return size;
918 }
919 
920 enum mlxsw_sp_dpipe_table_adj_match {
921 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX,
922 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE,
923 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX,
924 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT,
925 };
926 
927 enum mlxsw_sp_dpipe_table_adj_action {
928 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC,
929 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT,
930 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT,
931 };
932 
933 static void
934 mlxsw_sp_dpipe_table_adj_match_action_prepare(struct devlink_dpipe_match *matches,
935 					      struct devlink_dpipe_action *actions)
936 {
937 	struct devlink_dpipe_action *action;
938 	struct devlink_dpipe_match *match;
939 
940 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
941 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
942 	match->header = &mlxsw_sp_dpipe_header_metadata;
943 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
944 
945 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
946 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
947 	match->header = &mlxsw_sp_dpipe_header_metadata;
948 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
949 
950 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
951 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
952 	match->header = &mlxsw_sp_dpipe_header_metadata;
953 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
954 
955 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
956 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
957 	action->header = &devlink_dpipe_header_ethernet;
958 	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
959 
960 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
961 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
962 	action->header = &mlxsw_sp_dpipe_header_metadata;
963 	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
964 }
965 
966 static int
967 mlxsw_sp_dpipe_table_adj_entry_prepare(struct devlink_dpipe_entry *entry,
968 				       struct devlink_dpipe_value *match_values,
969 				       struct devlink_dpipe_match *matches,
970 				       struct devlink_dpipe_value *action_values,
971 				       struct devlink_dpipe_action *actions)
972 {	struct devlink_dpipe_value *action_value;
973 	struct devlink_dpipe_value *match_value;
974 	struct devlink_dpipe_action *action;
975 	struct devlink_dpipe_match *match;
976 
977 	entry->match_values = match_values;
978 	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT;
979 
980 	entry->action_values = action_values;
981 	entry->action_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT;
982 
983 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
984 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
985 
986 	match_value->match = match;
987 	match_value->value_size = sizeof(u32);
988 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
989 	if (!match_value->value)
990 		return -ENOMEM;
991 
992 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
993 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
994 
995 	match_value->match = match;
996 	match_value->value_size = sizeof(u32);
997 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
998 	if (!match_value->value)
999 		return -ENOMEM;
1000 
1001 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1002 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1003 
1004 	match_value->match = match;
1005 	match_value->value_size = sizeof(u32);
1006 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
1007 	if (!match_value->value)
1008 		return -ENOMEM;
1009 
1010 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1011 	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1012 
1013 	action_value->action = action;
1014 	action_value->value_size = sizeof(u64);
1015 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1016 	if (!action_value->value)
1017 		return -ENOMEM;
1018 
1019 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1020 	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1021 
1022 	action_value->action = action;
1023 	action_value->value_size = sizeof(u32);
1024 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1025 	if (!action_value->value)
1026 		return -ENOMEM;
1027 
1028 	return 0;
1029 }
1030 
1031 static void
1032 __mlxsw_sp_dpipe_table_adj_entry_fill(struct devlink_dpipe_entry *entry,
1033 				      u32 adj_index, u32 adj_size,
1034 				      u32 adj_hash_index, unsigned char *ha,
1035 				      struct mlxsw_sp_rif *rif)
1036 {
1037 	struct devlink_dpipe_value *value;
1038 	u32 *p_rif_value;
1039 	u32 *p_index;
1040 
1041 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
1042 	p_index = value->value;
1043 	*p_index = adj_index;
1044 
1045 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
1046 	p_index = value->value;
1047 	*p_index = adj_size;
1048 
1049 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1050 	p_index = value->value;
1051 	*p_index = adj_hash_index;
1052 
1053 	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1054 	ether_addr_copy(value->value, ha);
1055 
1056 	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1057 	p_rif_value = value->value;
1058 	*p_rif_value = mlxsw_sp_rif_index(rif);
1059 	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
1060 	value->mapping_valid = true;
1061 }
1062 
1063 static void mlxsw_sp_dpipe_table_adj_entry_fill(struct mlxsw_sp *mlxsw_sp,
1064 						struct mlxsw_sp_nexthop *nh,
1065 						struct devlink_dpipe_entry *entry)
1066 {
1067 	struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
1068 	unsigned char *ha = mlxsw_sp_nexthop_ha(nh);
1069 	u32 adj_hash_index = 0;
1070 	u32 adj_index = 0;
1071 	u32 adj_size = 0;
1072 	int err;
1073 
1074 	mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size, &adj_hash_index);
1075 	__mlxsw_sp_dpipe_table_adj_entry_fill(entry, adj_index, adj_size,
1076 					      adj_hash_index, ha, rif);
1077 	err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &entry->counter);
1078 	if (!err)
1079 		entry->counter_valid = true;
1080 }
1081 
1082 static int
1083 mlxsw_sp_dpipe_table_adj_entries_get(struct mlxsw_sp *mlxsw_sp,
1084 				     struct devlink_dpipe_entry *entry,
1085 				     bool counters_enabled,
1086 				     struct devlink_dpipe_dump_ctx *dump_ctx)
1087 {
1088 	struct mlxsw_sp_nexthop *nh;
1089 	int entry_index = 0;
1090 	int nh_count_max;
1091 	int nh_count = 0;
1092 	int nh_skip;
1093 	int j;
1094 	int err;
1095 
1096 	rtnl_lock();
1097 	nh_count_max = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1098 start_again:
1099 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
1100 	if (err)
1101 		goto err_ctx_prepare;
1102 	j = 0;
1103 	nh_skip = nh_count;
1104 	nh_count = 0;
1105 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1106 		if (!mlxsw_sp_nexthop_offload(nh) ||
1107 		    mlxsw_sp_nexthop_group_has_ipip(nh))
1108 			continue;
1109 
1110 		if (nh_count < nh_skip)
1111 			goto skip;
1112 
1113 		mlxsw_sp_dpipe_table_adj_entry_fill(mlxsw_sp, nh, entry);
1114 		entry->index = entry_index;
1115 		err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
1116 		if (err) {
1117 			if (err == -EMSGSIZE) {
1118 				if (!j)
1119 					goto err_entry_append;
1120 				break;
1121 			}
1122 			goto err_entry_append;
1123 		}
1124 		entry_index++;
1125 		j++;
1126 skip:
1127 		nh_count++;
1128 	}
1129 
1130 	devlink_dpipe_entry_ctx_close(dump_ctx);
1131 	if (nh_count != nh_count_max)
1132 		goto start_again;
1133 	rtnl_unlock();
1134 
1135 	return 0;
1136 
1137 err_ctx_prepare:
1138 err_entry_append:
1139 	rtnl_unlock();
1140 	return err;
1141 }
1142 
1143 static int
1144 mlxsw_sp_dpipe_table_adj_entries_dump(void *priv, bool counters_enabled,
1145 				      struct devlink_dpipe_dump_ctx *dump_ctx)
1146 {
1147 	struct devlink_dpipe_value action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1148 	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1149 	struct devlink_dpipe_action actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1150 	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1151 	struct devlink_dpipe_entry entry = {0};
1152 	struct mlxsw_sp *mlxsw_sp = priv;
1153 	int err;
1154 
1155 	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1156 			   sizeof(matches[0]));
1157 	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1158 				sizeof(match_values[0]));
1159 	memset(actions, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1160 			   sizeof(actions[0]));
1161 	memset(action_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1162 				 sizeof(action_values[0]));
1163 
1164 	mlxsw_sp_dpipe_table_adj_match_action_prepare(matches, actions);
1165 	err = mlxsw_sp_dpipe_table_adj_entry_prepare(&entry,
1166 						     match_values, matches,
1167 						     action_values, actions);
1168 	if (err)
1169 		goto out;
1170 
1171 	err = mlxsw_sp_dpipe_table_adj_entries_get(mlxsw_sp, &entry,
1172 						   counters_enabled, dump_ctx);
1173 out:
1174 	devlink_dpipe_entry_clear(&entry);
1175 	return err;
1176 }
1177 
1178 static int mlxsw_sp_dpipe_table_adj_counters_update(void *priv, bool enable)
1179 {
1180 	struct mlxsw_sp *mlxsw_sp = priv;
1181 	struct mlxsw_sp_nexthop *nh;
1182 	u32 adj_hash_index = 0;
1183 	u32 adj_index = 0;
1184 	u32 adj_size = 0;
1185 
1186 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1187 		if (!mlxsw_sp_nexthop_offload(nh) ||
1188 		    mlxsw_sp_nexthop_group_has_ipip(nh))
1189 			continue;
1190 
1191 		mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size,
1192 					 &adj_hash_index);
1193 		if (enable)
1194 			mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
1195 		else
1196 			mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
1197 		mlxsw_sp_nexthop_update(mlxsw_sp,
1198 					adj_index + adj_hash_index, nh);
1199 	}
1200 	return 0;
1201 }
1202 
1203 static u64
1204 mlxsw_sp_dpipe_table_adj_size_get(void *priv)
1205 {
1206 	struct mlxsw_sp *mlxsw_sp = priv;
1207 	u64 size;
1208 
1209 	rtnl_lock();
1210 	size = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1211 	rtnl_unlock();
1212 
1213 	return size;
1214 }
1215 
1216 static struct devlink_dpipe_table_ops mlxsw_sp_dpipe_table_adj_ops = {
1217 	.matches_dump = mlxsw_sp_dpipe_table_adj_matches_dump,
1218 	.actions_dump = mlxsw_sp_dpipe_table_adj_actions_dump,
1219 	.entries_dump = mlxsw_sp_dpipe_table_adj_entries_dump,
1220 	.counters_set_update = mlxsw_sp_dpipe_table_adj_counters_update,
1221 	.size_get = mlxsw_sp_dpipe_table_adj_size_get,
1222 };
1223 
1224 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ 1
1225 
1226 static int mlxsw_sp_dpipe_adj_table_init(struct mlxsw_sp *mlxsw_sp)
1227 {
1228 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1229 	int err;
1230 
1231 	err = devlink_dpipe_table_register(devlink,
1232 					   MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1233 					   &mlxsw_sp_dpipe_table_adj_ops,
1234 					   mlxsw_sp, false);
1235 	if (err)
1236 		return err;
1237 
1238 	err = devlink_dpipe_table_resource_set(devlink,
1239 					       MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1240 					       MLXSW_SP_RESOURCE_KVD_LINEAR,
1241 					       MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ);
1242 	if (err)
1243 		goto err_resource_set;
1244 
1245 	return 0;
1246 
1247 err_resource_set:
1248 	devlink_dpipe_table_unregister(devlink,
1249 				       MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1250 	return err;
1251 }
1252 
1253 static void mlxsw_sp_dpipe_adj_table_fini(struct mlxsw_sp *mlxsw_sp)
1254 {
1255 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1256 
1257 	devlink_dpipe_table_unregister(devlink,
1258 				       MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1259 }
1260 
1261 int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
1262 {
1263 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1264 	int err;
1265 
1266 	err = devlink_dpipe_headers_register(devlink,
1267 					     &mlxsw_sp_dpipe_headers);
1268 	if (err)
1269 		return err;
1270 	err = mlxsw_sp_dpipe_erif_table_init(mlxsw_sp);
1271 	if (err)
1272 		goto err_erif_table_init;
1273 
1274 	err = mlxsw_sp_dpipe_host4_table_init(mlxsw_sp);
1275 	if (err)
1276 		goto err_host4_table_init;
1277 
1278 	err = mlxsw_sp_dpipe_host6_table_init(mlxsw_sp);
1279 	if (err)
1280 		goto err_host6_table_init;
1281 
1282 	err = mlxsw_sp_dpipe_adj_table_init(mlxsw_sp);
1283 	if (err)
1284 		goto err_adj_table_init;
1285 
1286 	return 0;
1287 err_adj_table_init:
1288 	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1289 err_host6_table_init:
1290 	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1291 err_host4_table_init:
1292 	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1293 err_erif_table_init:
1294 	devlink_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core));
1295 	return err;
1296 }
1297 
1298 void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp)
1299 {
1300 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1301 
1302 	mlxsw_sp_dpipe_adj_table_fini(mlxsw_sp);
1303 	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1304 	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1305 	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1306 	devlink_dpipe_headers_unregister(devlink);
1307 }
1308