1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/mutex.h>
6 #include <net/devlink.h>
7 
8 #include "spectrum.h"
9 #include "spectrum_dpipe.h"
10 #include "spectrum_router.h"
11 
12 enum mlxsw_sp_field_metadata_id {
13 	MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
14 	MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
15 	MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
16 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
17 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
18 	MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
19 };
20 
21 static struct devlink_dpipe_field mlxsw_sp_dpipe_fields_metadata[] = {
22 	{
23 		.name = "erif_port",
24 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT,
25 		.bitwidth = 32,
26 		.mapping_type = DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX,
27 	},
28 	{
29 		.name = "l3_forward",
30 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD,
31 		.bitwidth = 1,
32 	},
33 	{
34 		.name = "l3_drop",
35 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP,
36 		.bitwidth = 1,
37 	},
38 	{
39 		.name = "adj_index",
40 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX,
41 		.bitwidth = 32,
42 	},
43 	{
44 		.name = "adj_size",
45 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE,
46 		.bitwidth = 32,
47 	},
48 	{
49 		.name = "adj_hash_index",
50 		.id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX,
51 		.bitwidth = 32,
52 	},
53 };
54 
55 enum mlxsw_sp_dpipe_header_id {
56 	MLXSW_SP_DPIPE_HEADER_METADATA,
57 };
58 
59 static struct devlink_dpipe_header mlxsw_sp_dpipe_header_metadata = {
60 	.name = "mlxsw_meta",
61 	.id = MLXSW_SP_DPIPE_HEADER_METADATA,
62 	.fields = mlxsw_sp_dpipe_fields_metadata,
63 	.fields_count = ARRAY_SIZE(mlxsw_sp_dpipe_fields_metadata),
64 };
65 
66 static struct devlink_dpipe_header *mlxsw_dpipe_headers[] = {
67 	&mlxsw_sp_dpipe_header_metadata,
68 	&devlink_dpipe_header_ethernet,
69 	&devlink_dpipe_header_ipv4,
70 	&devlink_dpipe_header_ipv6,
71 };
72 
73 static struct devlink_dpipe_headers mlxsw_sp_dpipe_headers = {
74 	.headers = mlxsw_dpipe_headers,
75 	.headers_count = ARRAY_SIZE(mlxsw_dpipe_headers),
76 };
77 
78 static int mlxsw_sp_dpipe_table_erif_actions_dump(void *priv,
79 						  struct sk_buff *skb)
80 {
81 	struct devlink_dpipe_action action = {0};
82 	int err;
83 
84 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
85 	action.header = &mlxsw_sp_dpipe_header_metadata;
86 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
87 
88 	err = devlink_dpipe_action_put(skb, &action);
89 	if (err)
90 		return err;
91 
92 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
93 	action.header = &mlxsw_sp_dpipe_header_metadata;
94 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_DROP;
95 
96 	return devlink_dpipe_action_put(skb, &action);
97 }
98 
99 static int mlxsw_sp_dpipe_table_erif_matches_dump(void *priv,
100 						  struct sk_buff *skb)
101 {
102 	struct devlink_dpipe_match match = {0};
103 
104 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
105 	match.header = &mlxsw_sp_dpipe_header_metadata;
106 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
107 
108 	return devlink_dpipe_match_put(skb, &match);
109 }
110 
111 static void
112 mlxsw_sp_erif_match_action_prepare(struct devlink_dpipe_match *match,
113 				   struct devlink_dpipe_action *action)
114 {
115 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
116 	action->header = &mlxsw_sp_dpipe_header_metadata;
117 	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_L3_FORWARD;
118 
119 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
120 	match->header = &mlxsw_sp_dpipe_header_metadata;
121 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
122 }
123 
124 static int mlxsw_sp_erif_entry_prepare(struct devlink_dpipe_entry *entry,
125 				       struct devlink_dpipe_value *match_value,
126 				       struct devlink_dpipe_match *match,
127 				       struct devlink_dpipe_value *action_value,
128 				       struct devlink_dpipe_action *action)
129 {
130 	entry->match_values = match_value;
131 	entry->match_values_count = 1;
132 
133 	entry->action_values = action_value;
134 	entry->action_values_count = 1;
135 
136 	match_value->match = match;
137 	match_value->value_size = sizeof(u32);
138 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
139 	if (!match_value->value)
140 		return -ENOMEM;
141 
142 	action_value->action = action;
143 	action_value->value_size = sizeof(u32);
144 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
145 	if (!action_value->value)
146 		goto err_action_alloc;
147 	return 0;
148 
149 err_action_alloc:
150 	kfree(match_value->value);
151 	return -ENOMEM;
152 }
153 
154 static int mlxsw_sp_erif_entry_get(struct mlxsw_sp *mlxsw_sp,
155 				   struct devlink_dpipe_entry *entry,
156 				   struct mlxsw_sp_rif *rif,
157 				   bool counters_enabled)
158 {
159 	u32 *action_value;
160 	u32 *rif_value;
161 	u64 cnt;
162 	int err;
163 
164 	/* Set Match RIF index */
165 	rif_value = entry->match_values->value;
166 	*rif_value = mlxsw_sp_rif_index(rif);
167 	entry->match_values->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
168 	entry->match_values->mapping_valid = true;
169 
170 	/* Set Action Forwarding */
171 	action_value = entry->action_values->value;
172 	*action_value = 1;
173 
174 	entry->counter_valid = false;
175 	entry->counter = 0;
176 	entry->index = mlxsw_sp_rif_index(rif);
177 
178 	if (!counters_enabled)
179 		return 0;
180 
181 	err = mlxsw_sp_rif_counter_value_get(mlxsw_sp, rif,
182 					     MLXSW_SP_RIF_COUNTER_EGRESS,
183 					     &cnt);
184 	if (!err) {
185 		entry->counter = cnt;
186 		entry->counter_valid = true;
187 	}
188 	return 0;
189 }
190 
191 static int
192 mlxsw_sp_dpipe_table_erif_entries_dump(void *priv, bool counters_enabled,
193 				       struct devlink_dpipe_dump_ctx *dump_ctx)
194 {
195 	struct devlink_dpipe_value match_value, action_value;
196 	struct devlink_dpipe_action action = {0};
197 	struct devlink_dpipe_match match = {0};
198 	struct devlink_dpipe_entry entry = {0};
199 	struct mlxsw_sp *mlxsw_sp = priv;
200 	unsigned int rif_count;
201 	int i, j;
202 	int err;
203 
204 	memset(&match_value, 0, sizeof(match_value));
205 	memset(&action_value, 0, sizeof(action_value));
206 
207 	mlxsw_sp_erif_match_action_prepare(&match, &action);
208 	err = mlxsw_sp_erif_entry_prepare(&entry, &match_value, &match,
209 					  &action_value, &action);
210 	if (err)
211 		return err;
212 
213 	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
214 	mutex_lock(&mlxsw_sp->router->lock);
215 	i = 0;
216 start_again:
217 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
218 	if (err)
219 		goto err_ctx_prepare;
220 	j = 0;
221 	for (; i < rif_count; i++) {
222 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
223 
224 		if (!rif || !mlxsw_sp_rif_dev(rif))
225 			continue;
226 		err = mlxsw_sp_erif_entry_get(mlxsw_sp, &entry, rif,
227 					      counters_enabled);
228 		if (err)
229 			goto err_entry_get;
230 		err = devlink_dpipe_entry_ctx_append(dump_ctx, &entry);
231 		if (err) {
232 			if (err == -EMSGSIZE) {
233 				if (!j)
234 					goto err_entry_append;
235 				break;
236 			}
237 			goto err_entry_append;
238 		}
239 		j++;
240 	}
241 
242 	devlink_dpipe_entry_ctx_close(dump_ctx);
243 	if (i != rif_count)
244 		goto start_again;
245 	mutex_unlock(&mlxsw_sp->router->lock);
246 
247 	devlink_dpipe_entry_clear(&entry);
248 	return 0;
249 err_entry_append:
250 err_entry_get:
251 err_ctx_prepare:
252 	mutex_unlock(&mlxsw_sp->router->lock);
253 	devlink_dpipe_entry_clear(&entry);
254 	return err;
255 }
256 
257 static int mlxsw_sp_dpipe_table_erif_counters_update(void *priv, bool enable)
258 {
259 	struct mlxsw_sp *mlxsw_sp = priv;
260 	int i;
261 
262 	mutex_lock(&mlxsw_sp->router->lock);
263 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
264 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
265 
266 		if (!rif)
267 			continue;
268 		if (enable)
269 			mlxsw_sp_rif_counter_alloc(rif,
270 						   MLXSW_SP_RIF_COUNTER_EGRESS);
271 		else
272 			mlxsw_sp_rif_counter_free(rif,
273 						  MLXSW_SP_RIF_COUNTER_EGRESS);
274 	}
275 	mutex_unlock(&mlxsw_sp->router->lock);
276 	return 0;
277 }
278 
279 static u64 mlxsw_sp_dpipe_table_erif_size_get(void *priv)
280 {
281 	struct mlxsw_sp *mlxsw_sp = priv;
282 
283 	return MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
284 }
285 
286 static struct devlink_dpipe_table_ops mlxsw_sp_erif_ops = {
287 	.matches_dump = mlxsw_sp_dpipe_table_erif_matches_dump,
288 	.actions_dump = mlxsw_sp_dpipe_table_erif_actions_dump,
289 	.entries_dump = mlxsw_sp_dpipe_table_erif_entries_dump,
290 	.counters_set_update = mlxsw_sp_dpipe_table_erif_counters_update,
291 	.size_get = mlxsw_sp_dpipe_table_erif_size_get,
292 };
293 
294 static int mlxsw_sp_dpipe_erif_table_init(struct mlxsw_sp *mlxsw_sp)
295 {
296 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
297 
298 	return devl_dpipe_table_register(devlink,
299 					 MLXSW_SP_DPIPE_TABLE_NAME_ERIF,
300 					 &mlxsw_sp_erif_ops,
301 					 mlxsw_sp, false);
302 }
303 
304 static void mlxsw_sp_dpipe_erif_table_fini(struct mlxsw_sp *mlxsw_sp)
305 {
306 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
307 
308 	devl_dpipe_table_unregister(devlink, MLXSW_SP_DPIPE_TABLE_NAME_ERIF);
309 }
310 
311 static int mlxsw_sp_dpipe_table_host_matches_dump(struct sk_buff *skb, int type)
312 {
313 	struct devlink_dpipe_match match = {0};
314 	int err;
315 
316 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
317 	match.header = &mlxsw_sp_dpipe_header_metadata;
318 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
319 
320 	err = devlink_dpipe_match_put(skb, &match);
321 	if (err)
322 		return err;
323 
324 	switch (type) {
325 	case AF_INET:
326 		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
327 		match.header = &devlink_dpipe_header_ipv4;
328 		match.field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
329 		break;
330 	case AF_INET6:
331 		match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
332 		match.header = &devlink_dpipe_header_ipv6;
333 		match.field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
334 		break;
335 	default:
336 		WARN_ON(1);
337 		return -EINVAL;
338 	}
339 
340 	return devlink_dpipe_match_put(skb, &match);
341 }
342 
343 static int
344 mlxsw_sp_dpipe_table_host4_matches_dump(void *priv, struct sk_buff *skb)
345 {
346 	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET);
347 }
348 
349 static int
350 mlxsw_sp_dpipe_table_host_actions_dump(void *priv, struct sk_buff *skb)
351 {
352 	struct devlink_dpipe_action action = {0};
353 
354 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
355 	action.header = &devlink_dpipe_header_ethernet;
356 	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
357 
358 	return devlink_dpipe_action_put(skb, &action);
359 }
360 
361 enum mlxsw_sp_dpipe_table_host_match {
362 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF,
363 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP,
364 	MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT,
365 };
366 
367 static void
368 mlxsw_sp_dpipe_table_host_match_action_prepare(struct devlink_dpipe_match *matches,
369 					       struct devlink_dpipe_action *action,
370 					       int type)
371 {
372 	struct devlink_dpipe_match *match;
373 
374 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
375 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
376 	match->header = &mlxsw_sp_dpipe_header_metadata;
377 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
378 
379 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
380 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
381 	switch (type) {
382 	case AF_INET:
383 		match->header = &devlink_dpipe_header_ipv4;
384 		match->field_id = DEVLINK_DPIPE_FIELD_IPV4_DST_IP;
385 		break;
386 	case AF_INET6:
387 		match->header = &devlink_dpipe_header_ipv6;
388 		match->field_id = DEVLINK_DPIPE_FIELD_IPV6_DST_IP;
389 		break;
390 	default:
391 		WARN_ON(1);
392 		return;
393 	}
394 
395 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
396 	action->header = &devlink_dpipe_header_ethernet;
397 	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
398 }
399 
400 static int
401 mlxsw_sp_dpipe_table_host_entry_prepare(struct devlink_dpipe_entry *entry,
402 					struct devlink_dpipe_value *match_values,
403 					struct devlink_dpipe_match *matches,
404 					struct devlink_dpipe_value *action_value,
405 					struct devlink_dpipe_action *action,
406 					int type)
407 {
408 	struct devlink_dpipe_value *match_value;
409 	struct devlink_dpipe_match *match;
410 
411 	entry->match_values = match_values;
412 	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT;
413 
414 	entry->action_values = action_value;
415 	entry->action_values_count = 1;
416 
417 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
418 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
419 
420 	match_value->match = match;
421 	match_value->value_size = sizeof(u32);
422 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
423 	if (!match_value->value)
424 		return -ENOMEM;
425 
426 	match = &matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
427 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
428 
429 	match_value->match = match;
430 	switch (type) {
431 	case AF_INET:
432 		match_value->value_size = sizeof(u32);
433 		break;
434 	case AF_INET6:
435 		match_value->value_size = sizeof(struct in6_addr);
436 		break;
437 	default:
438 		WARN_ON(1);
439 		return -EINVAL;
440 	}
441 
442 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
443 	if (!match_value->value)
444 		return -ENOMEM;
445 
446 	action_value->action = action;
447 	action_value->value_size = sizeof(u64);
448 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
449 	if (!action_value->value)
450 		return -ENOMEM;
451 
452 	return 0;
453 }
454 
455 static void
456 __mlxsw_sp_dpipe_table_host_entry_fill(struct devlink_dpipe_entry *entry,
457 				       struct mlxsw_sp_rif *rif,
458 				       unsigned char *ha, void *dip)
459 {
460 	struct devlink_dpipe_value *value;
461 	u32 *rif_value;
462 	u8 *ha_value;
463 
464 	/* Set Match RIF index */
465 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_RIF];
466 
467 	rif_value = value->value;
468 	*rif_value = mlxsw_sp_rif_index(rif);
469 	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
470 	value->mapping_valid = true;
471 
472 	/* Set Match DIP */
473 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_DIP];
474 	memcpy(value->value, dip, value->value_size);
475 
476 	/* Set Action DMAC */
477 	value = entry->action_values;
478 	ha_value = value->value;
479 	ether_addr_copy(ha_value, ha);
480 }
481 
482 static void
483 mlxsw_sp_dpipe_table_host4_entry_fill(struct devlink_dpipe_entry *entry,
484 				      struct mlxsw_sp_neigh_entry *neigh_entry,
485 				      struct mlxsw_sp_rif *rif)
486 {
487 	unsigned char *ha;
488 	u32 dip;
489 
490 	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
491 	dip = mlxsw_sp_neigh4_entry_dip(neigh_entry);
492 	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, &dip);
493 }
494 
495 static void
496 mlxsw_sp_dpipe_table_host6_entry_fill(struct devlink_dpipe_entry *entry,
497 				      struct mlxsw_sp_neigh_entry *neigh_entry,
498 				      struct mlxsw_sp_rif *rif)
499 {
500 	struct in6_addr *dip;
501 	unsigned char *ha;
502 
503 	ha = mlxsw_sp_neigh_entry_ha(neigh_entry);
504 	dip = mlxsw_sp_neigh6_entry_dip(neigh_entry);
505 
506 	__mlxsw_sp_dpipe_table_host_entry_fill(entry, rif, ha, dip);
507 }
508 
509 static void
510 mlxsw_sp_dpipe_table_host_entry_fill(struct mlxsw_sp *mlxsw_sp,
511 				     struct devlink_dpipe_entry *entry,
512 				     struct mlxsw_sp_neigh_entry *neigh_entry,
513 				     struct mlxsw_sp_rif *rif,
514 				     int type)
515 {
516 	int err;
517 
518 	switch (type) {
519 	case AF_INET:
520 		mlxsw_sp_dpipe_table_host4_entry_fill(entry, neigh_entry, rif);
521 		break;
522 	case AF_INET6:
523 		mlxsw_sp_dpipe_table_host6_entry_fill(entry, neigh_entry, rif);
524 		break;
525 	default:
526 		WARN_ON(1);
527 		return;
528 	}
529 
530 	err = mlxsw_sp_neigh_counter_get(mlxsw_sp, neigh_entry,
531 					 &entry->counter);
532 	if (!err)
533 		entry->counter_valid = true;
534 }
535 
536 static int
537 mlxsw_sp_dpipe_table_host_entries_get(struct mlxsw_sp *mlxsw_sp,
538 				      struct devlink_dpipe_entry *entry,
539 				      bool counters_enabled,
540 				      struct devlink_dpipe_dump_ctx *dump_ctx,
541 				      int type)
542 {
543 	int rif_neigh_count = 0;
544 	int rif_neigh_skip = 0;
545 	int neigh_count = 0;
546 	int rif_count;
547 	int i, j;
548 	int err;
549 
550 	mutex_lock(&mlxsw_sp->router->lock);
551 	i = 0;
552 	rif_count = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
553 start_again:
554 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
555 	if (err)
556 		goto err_ctx_prepare;
557 	j = 0;
558 	rif_neigh_skip = rif_neigh_count;
559 	for (; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
560 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
561 		struct mlxsw_sp_neigh_entry *neigh_entry;
562 
563 		if (!rif)
564 			continue;
565 
566 		rif_neigh_count = 0;
567 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
568 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
569 
570 			if (neigh_type != type)
571 				continue;
572 
573 			if (neigh_type == AF_INET6 &&
574 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
575 				continue;
576 
577 			if (rif_neigh_count < rif_neigh_skip)
578 				goto skip;
579 
580 			mlxsw_sp_dpipe_table_host_entry_fill(mlxsw_sp, entry,
581 							     neigh_entry, rif,
582 							     type);
583 			entry->index = neigh_count;
584 			err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
585 			if (err) {
586 				if (err == -EMSGSIZE) {
587 					if (!j)
588 						goto err_entry_append;
589 					else
590 						goto out;
591 				}
592 				goto err_entry_append;
593 			}
594 			neigh_count++;
595 			j++;
596 skip:
597 			rif_neigh_count++;
598 		}
599 		rif_neigh_skip = 0;
600 	}
601 out:
602 	devlink_dpipe_entry_ctx_close(dump_ctx);
603 	if (i != rif_count)
604 		goto start_again;
605 
606 	mutex_unlock(&mlxsw_sp->router->lock);
607 	return 0;
608 
609 err_ctx_prepare:
610 err_entry_append:
611 	mutex_unlock(&mlxsw_sp->router->lock);
612 	return err;
613 }
614 
615 static int
616 mlxsw_sp_dpipe_table_host_entries_dump(struct mlxsw_sp *mlxsw_sp,
617 				       bool counters_enabled,
618 				       struct devlink_dpipe_dump_ctx *dump_ctx,
619 				       int type)
620 {
621 	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
622 	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT];
623 	struct devlink_dpipe_value action_value;
624 	struct devlink_dpipe_action action = {0};
625 	struct devlink_dpipe_entry entry = {0};
626 	int err;
627 
628 	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
629 			   sizeof(matches[0]));
630 	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_HOST_MATCH_COUNT *
631 				sizeof(match_values[0]));
632 	memset(&action_value, 0, sizeof(action_value));
633 
634 	mlxsw_sp_dpipe_table_host_match_action_prepare(matches, &action, type);
635 	err = mlxsw_sp_dpipe_table_host_entry_prepare(&entry, match_values,
636 						      matches, &action_value,
637 						      &action, type);
638 	if (err)
639 		goto out;
640 
641 	err = mlxsw_sp_dpipe_table_host_entries_get(mlxsw_sp, &entry,
642 						    counters_enabled, dump_ctx,
643 						    type);
644 out:
645 	devlink_dpipe_entry_clear(&entry);
646 	return err;
647 }
648 
649 static int
650 mlxsw_sp_dpipe_table_host4_entries_dump(void *priv, bool counters_enabled,
651 					struct devlink_dpipe_dump_ctx *dump_ctx)
652 {
653 	struct mlxsw_sp *mlxsw_sp = priv;
654 
655 	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
656 						      counters_enabled,
657 						      dump_ctx, AF_INET);
658 }
659 
660 static void
661 mlxsw_sp_dpipe_table_host_counters_update(struct mlxsw_sp *mlxsw_sp,
662 					  bool enable, int type)
663 {
664 	int i;
665 
666 	mutex_lock(&mlxsw_sp->router->lock);
667 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
668 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
669 		struct mlxsw_sp_neigh_entry *neigh_entry;
670 
671 		if (!rif)
672 			continue;
673 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
674 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
675 
676 			if (neigh_type != type)
677 				continue;
678 
679 			if (neigh_type == AF_INET6 &&
680 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
681 				continue;
682 
683 			mlxsw_sp_neigh_entry_counter_update(mlxsw_sp,
684 							    neigh_entry,
685 							    enable);
686 		}
687 	}
688 	mutex_unlock(&mlxsw_sp->router->lock);
689 }
690 
691 static int mlxsw_sp_dpipe_table_host4_counters_update(void *priv, bool enable)
692 {
693 	struct mlxsw_sp *mlxsw_sp = priv;
694 
695 	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET);
696 	return 0;
697 }
698 
699 static u64
700 mlxsw_sp_dpipe_table_host_size_get(struct mlxsw_sp *mlxsw_sp, int type)
701 {
702 	u64 size = 0;
703 	int i;
704 
705 	mutex_lock(&mlxsw_sp->router->lock);
706 	for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
707 		struct mlxsw_sp_rif *rif = mlxsw_sp_rif_by_index(mlxsw_sp, i);
708 		struct mlxsw_sp_neigh_entry *neigh_entry;
709 
710 		if (!rif)
711 			continue;
712 		mlxsw_sp_rif_neigh_for_each(neigh_entry, rif) {
713 			int neigh_type = mlxsw_sp_neigh_entry_type(neigh_entry);
714 
715 			if (neigh_type != type)
716 				continue;
717 
718 			if (neigh_type == AF_INET6 &&
719 			    mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
720 				continue;
721 
722 			size++;
723 		}
724 	}
725 	mutex_unlock(&mlxsw_sp->router->lock);
726 
727 	return size;
728 }
729 
730 static u64 mlxsw_sp_dpipe_table_host4_size_get(void *priv)
731 {
732 	struct mlxsw_sp *mlxsw_sp = priv;
733 
734 	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET);
735 }
736 
737 static struct devlink_dpipe_table_ops mlxsw_sp_host4_ops = {
738 	.matches_dump = mlxsw_sp_dpipe_table_host4_matches_dump,
739 	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
740 	.entries_dump = mlxsw_sp_dpipe_table_host4_entries_dump,
741 	.counters_set_update = mlxsw_sp_dpipe_table_host4_counters_update,
742 	.size_get = mlxsw_sp_dpipe_table_host4_size_get,
743 };
744 
745 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4 1
746 
747 static int mlxsw_sp_dpipe_host4_table_init(struct mlxsw_sp *mlxsw_sp)
748 {
749 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
750 	int err;
751 
752 	err = devl_dpipe_table_register(devlink,
753 					MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
754 					&mlxsw_sp_host4_ops,
755 					mlxsw_sp, false);
756 	if (err)
757 		return err;
758 
759 	err = devl_dpipe_table_resource_set(devlink,
760 					    MLXSW_SP_DPIPE_TABLE_NAME_HOST4,
761 					    MLXSW_SP_RESOURCE_KVD_HASH_SINGLE,
762 					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST4);
763 	if (err)
764 		goto err_resource_set;
765 
766 	return 0;
767 
768 err_resource_set:
769 	devl_dpipe_table_unregister(devlink,
770 				    MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
771 	return err;
772 }
773 
774 static void mlxsw_sp_dpipe_host4_table_fini(struct mlxsw_sp *mlxsw_sp)
775 {
776 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
777 
778 	devl_dpipe_table_unregister(devlink,
779 				    MLXSW_SP_DPIPE_TABLE_NAME_HOST4);
780 }
781 
782 static int
783 mlxsw_sp_dpipe_table_host6_matches_dump(void *priv, struct sk_buff *skb)
784 {
785 	return mlxsw_sp_dpipe_table_host_matches_dump(skb, AF_INET6);
786 }
787 
788 static int
789 mlxsw_sp_dpipe_table_host6_entries_dump(void *priv, bool counters_enabled,
790 					struct devlink_dpipe_dump_ctx *dump_ctx)
791 {
792 	struct mlxsw_sp *mlxsw_sp = priv;
793 
794 	return mlxsw_sp_dpipe_table_host_entries_dump(mlxsw_sp,
795 						      counters_enabled,
796 						      dump_ctx, AF_INET6);
797 }
798 
799 static int mlxsw_sp_dpipe_table_host6_counters_update(void *priv, bool enable)
800 {
801 	struct mlxsw_sp *mlxsw_sp = priv;
802 
803 	mlxsw_sp_dpipe_table_host_counters_update(mlxsw_sp, enable, AF_INET6);
804 	return 0;
805 }
806 
807 static u64 mlxsw_sp_dpipe_table_host6_size_get(void *priv)
808 {
809 	struct mlxsw_sp *mlxsw_sp = priv;
810 
811 	return mlxsw_sp_dpipe_table_host_size_get(mlxsw_sp, AF_INET6);
812 }
813 
814 static struct devlink_dpipe_table_ops mlxsw_sp_host6_ops = {
815 	.matches_dump = mlxsw_sp_dpipe_table_host6_matches_dump,
816 	.actions_dump = mlxsw_sp_dpipe_table_host_actions_dump,
817 	.entries_dump = mlxsw_sp_dpipe_table_host6_entries_dump,
818 	.counters_set_update = mlxsw_sp_dpipe_table_host6_counters_update,
819 	.size_get = mlxsw_sp_dpipe_table_host6_size_get,
820 };
821 
822 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6 2
823 
824 static int mlxsw_sp_dpipe_host6_table_init(struct mlxsw_sp *mlxsw_sp)
825 {
826 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
827 	int err;
828 
829 	err = devl_dpipe_table_register(devlink,
830 					MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
831 					&mlxsw_sp_host6_ops,
832 					mlxsw_sp, false);
833 	if (err)
834 		return err;
835 
836 	err = devl_dpipe_table_resource_set(devlink,
837 					    MLXSW_SP_DPIPE_TABLE_NAME_HOST6,
838 					    MLXSW_SP_RESOURCE_KVD_HASH_DOUBLE,
839 					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_HOST6);
840 	if (err)
841 		goto err_resource_set;
842 
843 	return 0;
844 
845 err_resource_set:
846 	devl_dpipe_table_unregister(devlink,
847 				    MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
848 	return err;
849 }
850 
851 static void mlxsw_sp_dpipe_host6_table_fini(struct mlxsw_sp *mlxsw_sp)
852 {
853 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
854 
855 	devl_dpipe_table_unregister(devlink,
856 				    MLXSW_SP_DPIPE_TABLE_NAME_HOST6);
857 }
858 
859 static int mlxsw_sp_dpipe_table_adj_matches_dump(void *priv,
860 						 struct sk_buff *skb)
861 {
862 	struct devlink_dpipe_match match = {0};
863 	int err;
864 
865 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
866 	match.header = &mlxsw_sp_dpipe_header_metadata;
867 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
868 
869 	err = devlink_dpipe_match_put(skb, &match);
870 	if (err)
871 		return err;
872 
873 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
874 	match.header = &mlxsw_sp_dpipe_header_metadata;
875 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
876 
877 	err = devlink_dpipe_match_put(skb, &match);
878 	if (err)
879 		return err;
880 
881 	match.type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
882 	match.header = &mlxsw_sp_dpipe_header_metadata;
883 	match.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
884 
885 	return devlink_dpipe_match_put(skb, &match);
886 }
887 
888 static int mlxsw_sp_dpipe_table_adj_actions_dump(void *priv,
889 						 struct sk_buff *skb)
890 {
891 	struct devlink_dpipe_action action = {0};
892 	int err;
893 
894 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
895 	action.header = &devlink_dpipe_header_ethernet;
896 	action.field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
897 
898 	err = devlink_dpipe_action_put(skb, &action);
899 	if (err)
900 		return err;
901 
902 	action.type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
903 	action.header = &mlxsw_sp_dpipe_header_metadata;
904 	action.field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
905 
906 	return devlink_dpipe_action_put(skb, &action);
907 }
908 
909 static u64 mlxsw_sp_dpipe_table_adj_size(struct mlxsw_sp *mlxsw_sp)
910 {
911 	struct mlxsw_sp_nexthop *nh;
912 	u64 size = 0;
913 
914 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router)
915 		if (mlxsw_sp_nexthop_is_forward(nh) &&
916 		    !mlxsw_sp_nexthop_group_has_ipip(nh))
917 			size++;
918 	return size;
919 }
920 
921 enum mlxsw_sp_dpipe_table_adj_match {
922 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX,
923 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE,
924 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX,
925 	MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT,
926 };
927 
928 enum mlxsw_sp_dpipe_table_adj_action {
929 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC,
930 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT,
931 	MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT,
932 };
933 
934 static void
935 mlxsw_sp_dpipe_table_adj_match_action_prepare(struct devlink_dpipe_match *matches,
936 					      struct devlink_dpipe_action *actions)
937 {
938 	struct devlink_dpipe_action *action;
939 	struct devlink_dpipe_match *match;
940 
941 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
942 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
943 	match->header = &mlxsw_sp_dpipe_header_metadata;
944 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_INDEX;
945 
946 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
947 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
948 	match->header = &mlxsw_sp_dpipe_header_metadata;
949 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_SIZE;
950 
951 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
952 	match->type = DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT;
953 	match->header = &mlxsw_sp_dpipe_header_metadata;
954 	match->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ADJ_HASH_INDEX;
955 
956 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
957 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
958 	action->header = &devlink_dpipe_header_ethernet;
959 	action->field_id = DEVLINK_DPIPE_FIELD_ETHERNET_DST_MAC;
960 
961 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
962 	action->type = DEVLINK_DPIPE_ACTION_TYPE_FIELD_MODIFY;
963 	action->header = &mlxsw_sp_dpipe_header_metadata;
964 	action->field_id = MLXSW_SP_DPIPE_FIELD_METADATA_ERIF_PORT;
965 }
966 
967 static int
968 mlxsw_sp_dpipe_table_adj_entry_prepare(struct devlink_dpipe_entry *entry,
969 				       struct devlink_dpipe_value *match_values,
970 				       struct devlink_dpipe_match *matches,
971 				       struct devlink_dpipe_value *action_values,
972 				       struct devlink_dpipe_action *actions)
973 {	struct devlink_dpipe_value *action_value;
974 	struct devlink_dpipe_value *match_value;
975 	struct devlink_dpipe_action *action;
976 	struct devlink_dpipe_match *match;
977 
978 	entry->match_values = match_values;
979 	entry->match_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT;
980 
981 	entry->action_values = action_values;
982 	entry->action_values_count = MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT;
983 
984 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
985 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
986 
987 	match_value->match = match;
988 	match_value->value_size = sizeof(u32);
989 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
990 	if (!match_value->value)
991 		return -ENOMEM;
992 
993 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
994 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
995 
996 	match_value->match = match;
997 	match_value->value_size = sizeof(u32);
998 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
999 	if (!match_value->value)
1000 		return -ENOMEM;
1001 
1002 	match = &matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1003 	match_value = &match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1004 
1005 	match_value->match = match;
1006 	match_value->value_size = sizeof(u32);
1007 	match_value->value = kmalloc(match_value->value_size, GFP_KERNEL);
1008 	if (!match_value->value)
1009 		return -ENOMEM;
1010 
1011 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1012 	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1013 
1014 	action_value->action = action;
1015 	action_value->value_size = sizeof(u64);
1016 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1017 	if (!action_value->value)
1018 		return -ENOMEM;
1019 
1020 	action = &actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1021 	action_value = &action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1022 
1023 	action_value->action = action;
1024 	action_value->value_size = sizeof(u32);
1025 	action_value->value = kmalloc(action_value->value_size, GFP_KERNEL);
1026 	if (!action_value->value)
1027 		return -ENOMEM;
1028 
1029 	return 0;
1030 }
1031 
1032 static void
1033 __mlxsw_sp_dpipe_table_adj_entry_fill(struct devlink_dpipe_entry *entry,
1034 				      u32 adj_index, u32 adj_size,
1035 				      u32 adj_hash_index, unsigned char *ha,
1036 				      struct mlxsw_sp_rif *rif)
1037 {
1038 	struct devlink_dpipe_value *value;
1039 	u32 *p_rif_value;
1040 	u32 *p_index;
1041 
1042 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_INDEX];
1043 	p_index = value->value;
1044 	*p_index = adj_index;
1045 
1046 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_SIZE];
1047 	p_index = value->value;
1048 	*p_index = adj_size;
1049 
1050 	value = &entry->match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_HASH_INDEX];
1051 	p_index = value->value;
1052 	*p_index = adj_hash_index;
1053 
1054 	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_DST_MAC];
1055 	ether_addr_copy(value->value, ha);
1056 
1057 	value = &entry->action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_ERIF_PORT];
1058 	p_rif_value = value->value;
1059 	*p_rif_value = mlxsw_sp_rif_index(rif);
1060 	value->mapping_value = mlxsw_sp_rif_dev_ifindex(rif);
1061 	value->mapping_valid = true;
1062 }
1063 
1064 static void mlxsw_sp_dpipe_table_adj_entry_fill(struct mlxsw_sp *mlxsw_sp,
1065 						struct mlxsw_sp_nexthop *nh,
1066 						struct devlink_dpipe_entry *entry)
1067 {
1068 	struct mlxsw_sp_rif *rif = mlxsw_sp_nexthop_rif(nh);
1069 	unsigned char *ha = mlxsw_sp_nexthop_ha(nh);
1070 	u32 adj_hash_index = 0;
1071 	u32 adj_index = 0;
1072 	u32 adj_size = 0;
1073 	int err;
1074 
1075 	mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size, &adj_hash_index);
1076 	__mlxsw_sp_dpipe_table_adj_entry_fill(entry, adj_index, adj_size,
1077 					      adj_hash_index, ha, rif);
1078 	err = mlxsw_sp_nexthop_counter_get(mlxsw_sp, nh, &entry->counter);
1079 	if (!err)
1080 		entry->counter_valid = true;
1081 }
1082 
1083 static int
1084 mlxsw_sp_dpipe_table_adj_entries_get(struct mlxsw_sp *mlxsw_sp,
1085 				     struct devlink_dpipe_entry *entry,
1086 				     bool counters_enabled,
1087 				     struct devlink_dpipe_dump_ctx *dump_ctx)
1088 {
1089 	struct mlxsw_sp_nexthop *nh;
1090 	int entry_index = 0;
1091 	int nh_count_max;
1092 	int nh_count = 0;
1093 	int nh_skip;
1094 	int j;
1095 	int err;
1096 
1097 	mutex_lock(&mlxsw_sp->router->lock);
1098 	nh_count_max = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1099 start_again:
1100 	err = devlink_dpipe_entry_ctx_prepare(dump_ctx);
1101 	if (err)
1102 		goto err_ctx_prepare;
1103 	j = 0;
1104 	nh_skip = nh_count;
1105 	nh_count = 0;
1106 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1107 		if (!mlxsw_sp_nexthop_is_forward(nh) ||
1108 		    mlxsw_sp_nexthop_group_has_ipip(nh))
1109 			continue;
1110 
1111 		if (nh_count < nh_skip)
1112 			goto skip;
1113 
1114 		mlxsw_sp_dpipe_table_adj_entry_fill(mlxsw_sp, nh, entry);
1115 		entry->index = entry_index;
1116 		err = devlink_dpipe_entry_ctx_append(dump_ctx, entry);
1117 		if (err) {
1118 			if (err == -EMSGSIZE) {
1119 				if (!j)
1120 					goto err_entry_append;
1121 				break;
1122 			}
1123 			goto err_entry_append;
1124 		}
1125 		entry_index++;
1126 		j++;
1127 skip:
1128 		nh_count++;
1129 	}
1130 
1131 	devlink_dpipe_entry_ctx_close(dump_ctx);
1132 	if (nh_count != nh_count_max)
1133 		goto start_again;
1134 	mutex_unlock(&mlxsw_sp->router->lock);
1135 
1136 	return 0;
1137 
1138 err_ctx_prepare:
1139 err_entry_append:
1140 	mutex_unlock(&mlxsw_sp->router->lock);
1141 	return err;
1142 }
1143 
1144 static int
1145 mlxsw_sp_dpipe_table_adj_entries_dump(void *priv, bool counters_enabled,
1146 				      struct devlink_dpipe_dump_ctx *dump_ctx)
1147 {
1148 	struct devlink_dpipe_value action_values[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1149 	struct devlink_dpipe_value match_values[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1150 	struct devlink_dpipe_action actions[MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT];
1151 	struct devlink_dpipe_match matches[MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT];
1152 	struct devlink_dpipe_entry entry = {0};
1153 	struct mlxsw_sp *mlxsw_sp = priv;
1154 	int err;
1155 
1156 	memset(matches, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1157 			   sizeof(matches[0]));
1158 	memset(match_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_MATCH_COUNT *
1159 				sizeof(match_values[0]));
1160 	memset(actions, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1161 			   sizeof(actions[0]));
1162 	memset(action_values, 0, MLXSW_SP_DPIPE_TABLE_ADJ_ACTION_COUNT *
1163 				 sizeof(action_values[0]));
1164 
1165 	mlxsw_sp_dpipe_table_adj_match_action_prepare(matches, actions);
1166 	err = mlxsw_sp_dpipe_table_adj_entry_prepare(&entry,
1167 						     match_values, matches,
1168 						     action_values, actions);
1169 	if (err)
1170 		goto out;
1171 
1172 	err = mlxsw_sp_dpipe_table_adj_entries_get(mlxsw_sp, &entry,
1173 						   counters_enabled, dump_ctx);
1174 out:
1175 	devlink_dpipe_entry_clear(&entry);
1176 	return err;
1177 }
1178 
1179 static int mlxsw_sp_dpipe_table_adj_counters_update(void *priv, bool enable)
1180 {
1181 	char ratr_pl[MLXSW_REG_RATR_LEN];
1182 	struct mlxsw_sp *mlxsw_sp = priv;
1183 	struct mlxsw_sp_nexthop *nh;
1184 	u32 adj_hash_index = 0;
1185 	u32 adj_index = 0;
1186 	u32 adj_size = 0;
1187 
1188 	mlxsw_sp_nexthop_for_each(nh, mlxsw_sp->router) {
1189 		if (!mlxsw_sp_nexthop_is_forward(nh) ||
1190 		    mlxsw_sp_nexthop_group_has_ipip(nh))
1191 			continue;
1192 
1193 		mlxsw_sp_nexthop_indexes(nh, &adj_index, &adj_size,
1194 					 &adj_hash_index);
1195 		if (enable)
1196 			mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
1197 		else
1198 			mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
1199 		mlxsw_sp_nexthop_eth_update(mlxsw_sp,
1200 					    adj_index + adj_hash_index, nh,
1201 					    true, ratr_pl);
1202 	}
1203 	return 0;
1204 }
1205 
1206 static u64
1207 mlxsw_sp_dpipe_table_adj_size_get(void *priv)
1208 {
1209 	struct mlxsw_sp *mlxsw_sp = priv;
1210 	u64 size;
1211 
1212 	mutex_lock(&mlxsw_sp->router->lock);
1213 	size = mlxsw_sp_dpipe_table_adj_size(mlxsw_sp);
1214 	mutex_unlock(&mlxsw_sp->router->lock);
1215 
1216 	return size;
1217 }
1218 
1219 static struct devlink_dpipe_table_ops mlxsw_sp_dpipe_table_adj_ops = {
1220 	.matches_dump = mlxsw_sp_dpipe_table_adj_matches_dump,
1221 	.actions_dump = mlxsw_sp_dpipe_table_adj_actions_dump,
1222 	.entries_dump = mlxsw_sp_dpipe_table_adj_entries_dump,
1223 	.counters_set_update = mlxsw_sp_dpipe_table_adj_counters_update,
1224 	.size_get = mlxsw_sp_dpipe_table_adj_size_get,
1225 };
1226 
1227 #define MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ 1
1228 
1229 static int mlxsw_sp_dpipe_adj_table_init(struct mlxsw_sp *mlxsw_sp)
1230 {
1231 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1232 	int err;
1233 
1234 	err = devl_dpipe_table_register(devlink,
1235 					MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1236 					&mlxsw_sp_dpipe_table_adj_ops,
1237 					mlxsw_sp, false);
1238 	if (err)
1239 		return err;
1240 
1241 	err = devl_dpipe_table_resource_set(devlink,
1242 					    MLXSW_SP_DPIPE_TABLE_NAME_ADJ,
1243 					    MLXSW_SP_RESOURCE_KVD_LINEAR,
1244 					    MLXSW_SP_DPIPE_TABLE_RESOURCE_UNIT_ADJ);
1245 	if (err)
1246 		goto err_resource_set;
1247 
1248 	return 0;
1249 
1250 err_resource_set:
1251 	devl_dpipe_table_unregister(devlink,
1252 				    MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1253 	return err;
1254 }
1255 
1256 static void mlxsw_sp_dpipe_adj_table_fini(struct mlxsw_sp *mlxsw_sp)
1257 {
1258 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1259 
1260 	devl_dpipe_table_unregister(devlink,
1261 				    MLXSW_SP_DPIPE_TABLE_NAME_ADJ);
1262 }
1263 
1264 int mlxsw_sp_dpipe_init(struct mlxsw_sp *mlxsw_sp)
1265 {
1266 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1267 	int err;
1268 
1269 	devl_dpipe_headers_register(devlink, &mlxsw_sp_dpipe_headers);
1270 
1271 	err = mlxsw_sp_dpipe_erif_table_init(mlxsw_sp);
1272 	if (err)
1273 		goto err_erif_table_init;
1274 
1275 	err = mlxsw_sp_dpipe_host4_table_init(mlxsw_sp);
1276 	if (err)
1277 		goto err_host4_table_init;
1278 
1279 	err = mlxsw_sp_dpipe_host6_table_init(mlxsw_sp);
1280 	if (err)
1281 		goto err_host6_table_init;
1282 
1283 	err = mlxsw_sp_dpipe_adj_table_init(mlxsw_sp);
1284 	if (err)
1285 		goto err_adj_table_init;
1286 
1287 	return 0;
1288 err_adj_table_init:
1289 	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1290 err_host6_table_init:
1291 	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1292 err_host4_table_init:
1293 	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1294 err_erif_table_init:
1295 	devl_dpipe_headers_unregister(priv_to_devlink(mlxsw_sp->core));
1296 	return err;
1297 }
1298 
1299 void mlxsw_sp_dpipe_fini(struct mlxsw_sp *mlxsw_sp)
1300 {
1301 	struct devlink *devlink = priv_to_devlink(mlxsw_sp->core);
1302 
1303 	mlxsw_sp_dpipe_adj_table_fini(mlxsw_sp);
1304 	mlxsw_sp_dpipe_host6_table_fini(mlxsw_sp);
1305 	mlxsw_sp_dpipe_host4_table_fini(mlxsw_sp);
1306 	mlxsw_sp_dpipe_erif_table_fini(mlxsw_sp);
1307 	devl_dpipe_headers_unregister(devlink);
1308 }
1309