1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/slab.h>
6 #include <linux/errno.h>
7 #include <linux/bitops.h>
8 #include <linux/list.h>
9 #include <linux/rhashtable.h>
10 #include <linux/netdevice.h>
11 #include <linux/mutex.h>
12 #include <trace/events/mlxsw.h>
13 
14 #include "reg.h"
15 #include "core.h"
16 #include "resources.h"
17 #include "spectrum.h"
18 #include "spectrum_acl_tcam.h"
19 #include "core_acl_flex_keys.h"
20 
21 size_t mlxsw_sp_acl_tcam_priv_size(struct mlxsw_sp *mlxsw_sp)
22 {
23 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
24 
25 	return ops->priv_size;
26 }
27 
28 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT 5000 /* ms */
29 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN 3000 /* ms */
30 #define MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS 100 /* number of entries */
31 
32 int mlxsw_sp_acl_tcam_init(struct mlxsw_sp *mlxsw_sp,
33 			   struct mlxsw_sp_acl_tcam *tcam)
34 {
35 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
36 	u64 max_tcam_regions;
37 	u64 max_regions;
38 	u64 max_groups;
39 	size_t alloc_size;
40 	int err;
41 
42 	mutex_init(&tcam->lock);
43 	tcam->vregion_rehash_intrvl =
44 			MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_DFLT;
45 	INIT_LIST_HEAD(&tcam->vregion_list);
46 
47 	max_tcam_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core,
48 					      ACL_MAX_TCAM_REGIONS);
49 	max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
50 
51 	/* Use 1:1 mapping between ACL region and TCAM region */
52 	if (max_tcam_regions < max_regions)
53 		max_regions = max_tcam_regions;
54 
55 	alloc_size = sizeof(tcam->used_regions[0]) * BITS_TO_LONGS(max_regions);
56 	tcam->used_regions = kzalloc(alloc_size, GFP_KERNEL);
57 	if (!tcam->used_regions)
58 		return -ENOMEM;
59 	tcam->max_regions = max_regions;
60 
61 	max_groups = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_GROUPS);
62 	alloc_size = sizeof(tcam->used_groups[0]) * BITS_TO_LONGS(max_groups);
63 	tcam->used_groups = kzalloc(alloc_size, GFP_KERNEL);
64 	if (!tcam->used_groups) {
65 		err = -ENOMEM;
66 		goto err_alloc_used_groups;
67 	}
68 	tcam->max_groups = max_groups;
69 	tcam->max_group_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
70 						 ACL_MAX_GROUP_SIZE);
71 
72 	err = ops->init(mlxsw_sp, tcam->priv, tcam);
73 	if (err)
74 		goto err_tcam_init;
75 
76 	return 0;
77 
78 err_tcam_init:
79 	kfree(tcam->used_groups);
80 err_alloc_used_groups:
81 	kfree(tcam->used_regions);
82 	return err;
83 }
84 
85 void mlxsw_sp_acl_tcam_fini(struct mlxsw_sp *mlxsw_sp,
86 			    struct mlxsw_sp_acl_tcam *tcam)
87 {
88 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
89 
90 	mutex_destroy(&tcam->lock);
91 	ops->fini(mlxsw_sp, tcam->priv);
92 	kfree(tcam->used_groups);
93 	kfree(tcam->used_regions);
94 }
95 
96 int mlxsw_sp_acl_tcam_priority_get(struct mlxsw_sp *mlxsw_sp,
97 				   struct mlxsw_sp_acl_rule_info *rulei,
98 				   u32 *priority, bool fillup_priority)
99 {
100 	u64 max_priority;
101 
102 	if (!fillup_priority) {
103 		*priority = 0;
104 		return 0;
105 	}
106 
107 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, KVD_SIZE))
108 		return -EIO;
109 
110 	/* Priority range is 1..cap_kvd_size-1. */
111 	max_priority = MLXSW_CORE_RES_GET(mlxsw_sp->core, KVD_SIZE) - 1;
112 	if (rulei->priority >= max_priority)
113 		return -EINVAL;
114 
115 	/* Unlike in TC, in HW, higher number means higher priority. */
116 	*priority = max_priority - rulei->priority;
117 	return 0;
118 }
119 
120 static int mlxsw_sp_acl_tcam_region_id_get(struct mlxsw_sp_acl_tcam *tcam,
121 					   u16 *p_id)
122 {
123 	u16 id;
124 
125 	id = find_first_zero_bit(tcam->used_regions, tcam->max_regions);
126 	if (id < tcam->max_regions) {
127 		__set_bit(id, tcam->used_regions);
128 		*p_id = id;
129 		return 0;
130 	}
131 	return -ENOBUFS;
132 }
133 
134 static void mlxsw_sp_acl_tcam_region_id_put(struct mlxsw_sp_acl_tcam *tcam,
135 					    u16 id)
136 {
137 	__clear_bit(id, tcam->used_regions);
138 }
139 
140 static int mlxsw_sp_acl_tcam_group_id_get(struct mlxsw_sp_acl_tcam *tcam,
141 					  u16 *p_id)
142 {
143 	u16 id;
144 
145 	id = find_first_zero_bit(tcam->used_groups, tcam->max_groups);
146 	if (id < tcam->max_groups) {
147 		__set_bit(id, tcam->used_groups);
148 		*p_id = id;
149 		return 0;
150 	}
151 	return -ENOBUFS;
152 }
153 
154 static void mlxsw_sp_acl_tcam_group_id_put(struct mlxsw_sp_acl_tcam *tcam,
155 					   u16 id)
156 {
157 	__clear_bit(id, tcam->used_groups);
158 }
159 
160 struct mlxsw_sp_acl_tcam_pattern {
161 	const enum mlxsw_afk_element *elements;
162 	unsigned int elements_count;
163 };
164 
165 struct mlxsw_sp_acl_tcam_group {
166 	struct mlxsw_sp_acl_tcam *tcam;
167 	u16 id;
168 	struct mutex lock; /* guards region list updates */
169 	struct list_head region_list;
170 	unsigned int region_count;
171 };
172 
173 struct mlxsw_sp_acl_tcam_vgroup {
174 	struct mlxsw_sp_acl_tcam_group group;
175 	struct list_head vregion_list;
176 	struct rhashtable vchunk_ht;
177 	const struct mlxsw_sp_acl_tcam_pattern *patterns;
178 	unsigned int patterns_count;
179 	bool tmplt_elusage_set;
180 	struct mlxsw_afk_element_usage tmplt_elusage;
181 	bool vregion_rehash_enabled;
182 };
183 
184 struct mlxsw_sp_acl_tcam_rehash_ctx {
185 	void *hints_priv;
186 	bool this_is_rollback;
187 	struct mlxsw_sp_acl_tcam_vchunk *current_vchunk; /* vchunk being
188 							  * currently migrated.
189 							  */
190 	struct mlxsw_sp_acl_tcam_ventry *start_ventry; /* ventry to start
191 							* migration from in
192 							* a vchunk being
193 							* currently migrated.
194 							*/
195 	struct mlxsw_sp_acl_tcam_ventry *stop_ventry; /* ventry to stop
196 						       * migration at
197 						       * a vchunk being
198 						       * currently migrated.
199 						       */
200 };
201 
202 struct mlxsw_sp_acl_tcam_vregion {
203 	struct mutex lock; /* Protects consistency of region, region2 pointers
204 			    * and vchunk_list.
205 			    */
206 	struct mlxsw_sp_acl_tcam_region *region;
207 	struct mlxsw_sp_acl_tcam_region *region2; /* Used during migration */
208 	struct list_head list; /* Member of a TCAM group */
209 	struct list_head tlist; /* Member of a TCAM */
210 	struct list_head vchunk_list; /* List of vchunks under this vregion */
211 	struct mlxsw_afk_key_info *key_info;
212 	struct mlxsw_sp_acl_tcam *tcam;
213 	struct mlxsw_sp_acl_tcam_vgroup *vgroup;
214 	struct {
215 		struct delayed_work dw;
216 		struct mlxsw_sp_acl_tcam_rehash_ctx ctx;
217 	} rehash;
218 	struct mlxsw_sp *mlxsw_sp;
219 	bool failed_rollback; /* Indicates failed rollback during migration */
220 	unsigned int ref_count;
221 };
222 
223 struct mlxsw_sp_acl_tcam_vchunk;
224 
225 struct mlxsw_sp_acl_tcam_chunk {
226 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
227 	struct mlxsw_sp_acl_tcam_region *region;
228 	unsigned long priv[0];
229 	/* priv has to be always the last item */
230 };
231 
232 struct mlxsw_sp_acl_tcam_vchunk {
233 	struct mlxsw_sp_acl_tcam_chunk *chunk;
234 	struct mlxsw_sp_acl_tcam_chunk *chunk2; /* Used during migration */
235 	struct list_head list; /* Member of a TCAM vregion */
236 	struct rhash_head ht_node; /* Member of a chunk HT */
237 	struct list_head ventry_list;
238 	unsigned int priority; /* Priority within the vregion and group */
239 	struct mlxsw_sp_acl_tcam_vgroup *vgroup;
240 	struct mlxsw_sp_acl_tcam_vregion *vregion;
241 	unsigned int ref_count;
242 };
243 
244 struct mlxsw_sp_acl_tcam_entry {
245 	struct mlxsw_sp_acl_tcam_ventry *ventry;
246 	struct mlxsw_sp_acl_tcam_chunk *chunk;
247 	unsigned long priv[0];
248 	/* priv has to be always the last item */
249 };
250 
251 struct mlxsw_sp_acl_tcam_ventry {
252 	struct mlxsw_sp_acl_tcam_entry *entry;
253 	struct list_head list; /* Member of a TCAM vchunk */
254 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
255 	struct mlxsw_sp_acl_rule_info *rulei;
256 };
257 
258 static const struct rhashtable_params mlxsw_sp_acl_tcam_vchunk_ht_params = {
259 	.key_len = sizeof(unsigned int),
260 	.key_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, priority),
261 	.head_offset = offsetof(struct mlxsw_sp_acl_tcam_vchunk, ht_node),
262 	.automatic_shrinking = true,
263 };
264 
265 static int mlxsw_sp_acl_tcam_group_update(struct mlxsw_sp *mlxsw_sp,
266 					  struct mlxsw_sp_acl_tcam_group *group)
267 {
268 	struct mlxsw_sp_acl_tcam_region *region;
269 	char pagt_pl[MLXSW_REG_PAGT_LEN];
270 	int acl_index = 0;
271 
272 	mlxsw_reg_pagt_pack(pagt_pl, group->id);
273 	list_for_each_entry(region, &group->region_list, list) {
274 		bool multi = false;
275 
276 		/* Check if the next entry in the list has the same vregion. */
277 		if (region->list.next != &group->region_list &&
278 		    list_next_entry(region, list)->vregion == region->vregion)
279 			multi = true;
280 		mlxsw_reg_pagt_acl_id_pack(pagt_pl, acl_index++,
281 					   region->id, multi);
282 	}
283 	mlxsw_reg_pagt_size_set(pagt_pl, acl_index);
284 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pagt), pagt_pl);
285 }
286 
287 static int
288 mlxsw_sp_acl_tcam_group_add(struct mlxsw_sp_acl_tcam *tcam,
289 			    struct mlxsw_sp_acl_tcam_group *group)
290 {
291 	int err;
292 
293 	group->tcam = tcam;
294 	mutex_init(&group->lock);
295 	INIT_LIST_HEAD(&group->region_list);
296 
297 	err = mlxsw_sp_acl_tcam_group_id_get(tcam, &group->id);
298 	if (err)
299 		return err;
300 
301 	return 0;
302 }
303 
304 static void mlxsw_sp_acl_tcam_group_del(struct mlxsw_sp_acl_tcam_group *group)
305 {
306 	struct mlxsw_sp_acl_tcam *tcam = group->tcam;
307 
308 	mutex_destroy(&group->lock);
309 	mlxsw_sp_acl_tcam_group_id_put(tcam, group->id);
310 	WARN_ON(!list_empty(&group->region_list));
311 }
312 
313 static int
314 mlxsw_sp_acl_tcam_vgroup_add(struct mlxsw_sp *mlxsw_sp,
315 			     struct mlxsw_sp_acl_tcam *tcam,
316 			     struct mlxsw_sp_acl_tcam_vgroup *vgroup,
317 			     const struct mlxsw_sp_acl_tcam_pattern *patterns,
318 			     unsigned int patterns_count,
319 			     struct mlxsw_afk_element_usage *tmplt_elusage,
320 			     bool vregion_rehash_enabled)
321 {
322 	int err;
323 
324 	vgroup->patterns = patterns;
325 	vgroup->patterns_count = patterns_count;
326 	vgroup->vregion_rehash_enabled = vregion_rehash_enabled;
327 
328 	if (tmplt_elusage) {
329 		vgroup->tmplt_elusage_set = true;
330 		memcpy(&vgroup->tmplt_elusage, tmplt_elusage,
331 		       sizeof(vgroup->tmplt_elusage));
332 	}
333 	INIT_LIST_HEAD(&vgroup->vregion_list);
334 
335 	err = mlxsw_sp_acl_tcam_group_add(tcam, &vgroup->group);
336 	if (err)
337 		return err;
338 
339 	err = rhashtable_init(&vgroup->vchunk_ht,
340 			      &mlxsw_sp_acl_tcam_vchunk_ht_params);
341 	if (err)
342 		goto err_rhashtable_init;
343 
344 	return 0;
345 
346 err_rhashtable_init:
347 	mlxsw_sp_acl_tcam_group_del(&vgroup->group);
348 	return err;
349 }
350 
351 static void
352 mlxsw_sp_acl_tcam_vgroup_del(struct mlxsw_sp_acl_tcam_vgroup *vgroup)
353 {
354 	rhashtable_destroy(&vgroup->vchunk_ht);
355 	mlxsw_sp_acl_tcam_group_del(&vgroup->group);
356 	WARN_ON(!list_empty(&vgroup->vregion_list));
357 }
358 
359 static int
360 mlxsw_sp_acl_tcam_group_bind(struct mlxsw_sp *mlxsw_sp,
361 			     struct mlxsw_sp_acl_tcam_group *group,
362 			     struct mlxsw_sp_port *mlxsw_sp_port,
363 			     bool ingress)
364 {
365 	char ppbt_pl[MLXSW_REG_PPBT_LEN];
366 
367 	mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
368 					       MLXSW_REG_PXBT_E_EACL,
369 			    MLXSW_REG_PXBT_OP_BIND, mlxsw_sp_port->local_port,
370 			    group->id);
371 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
372 }
373 
374 static void
375 mlxsw_sp_acl_tcam_group_unbind(struct mlxsw_sp *mlxsw_sp,
376 			       struct mlxsw_sp_acl_tcam_group *group,
377 			       struct mlxsw_sp_port *mlxsw_sp_port,
378 			       bool ingress)
379 {
380 	char ppbt_pl[MLXSW_REG_PPBT_LEN];
381 
382 	mlxsw_reg_ppbt_pack(ppbt_pl, ingress ? MLXSW_REG_PXBT_E_IACL :
383 					       MLXSW_REG_PXBT_E_EACL,
384 			    MLXSW_REG_PXBT_OP_UNBIND, mlxsw_sp_port->local_port,
385 			    group->id);
386 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ppbt), ppbt_pl);
387 }
388 
389 static u16
390 mlxsw_sp_acl_tcam_group_id(struct mlxsw_sp_acl_tcam_group *group)
391 {
392 	return group->id;
393 }
394 
395 static unsigned int
396 mlxsw_sp_acl_tcam_vregion_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
397 {
398 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
399 
400 	if (list_empty(&vregion->vchunk_list))
401 		return 0;
402 	/* As a priority of a vregion, return priority of the first vchunk */
403 	vchunk = list_first_entry(&vregion->vchunk_list,
404 				  typeof(*vchunk), list);
405 	return vchunk->priority;
406 }
407 
408 static unsigned int
409 mlxsw_sp_acl_tcam_vregion_max_prio(struct mlxsw_sp_acl_tcam_vregion *vregion)
410 {
411 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
412 
413 	if (list_empty(&vregion->vchunk_list))
414 		return 0;
415 	vchunk = list_last_entry(&vregion->vchunk_list,
416 				 typeof(*vchunk), list);
417 	return vchunk->priority;
418 }
419 
420 static int
421 mlxsw_sp_acl_tcam_group_region_attach(struct mlxsw_sp *mlxsw_sp,
422 				      struct mlxsw_sp_acl_tcam_group *group,
423 				      struct mlxsw_sp_acl_tcam_region *region,
424 				      unsigned int priority,
425 				      struct mlxsw_sp_acl_tcam_region *next_region)
426 {
427 	struct mlxsw_sp_acl_tcam_region *region2;
428 	struct list_head *pos;
429 	int err;
430 
431 	mutex_lock(&group->lock);
432 	if (group->region_count == group->tcam->max_group_size) {
433 		err = -ENOBUFS;
434 		goto err_region_count_check;
435 	}
436 
437 	if (next_region) {
438 		/* If the next region is defined, place the new one
439 		 * before it. The next one is a sibling.
440 		 */
441 		pos = &next_region->list;
442 	} else {
443 		/* Position the region inside the list according to priority */
444 		list_for_each(pos, &group->region_list) {
445 			region2 = list_entry(pos, typeof(*region2), list);
446 			if (mlxsw_sp_acl_tcam_vregion_prio(region2->vregion) >
447 			    priority)
448 				break;
449 		}
450 	}
451 	list_add_tail(&region->list, pos);
452 	region->group = group;
453 
454 	err = mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
455 	if (err)
456 		goto err_group_update;
457 
458 	group->region_count++;
459 	mutex_unlock(&group->lock);
460 	return 0;
461 
462 err_group_update:
463 	list_del(&region->list);
464 err_region_count_check:
465 	mutex_unlock(&group->lock);
466 	return err;
467 }
468 
469 static void
470 mlxsw_sp_acl_tcam_group_region_detach(struct mlxsw_sp *mlxsw_sp,
471 				      struct mlxsw_sp_acl_tcam_region *region)
472 {
473 	struct mlxsw_sp_acl_tcam_group *group = region->group;
474 
475 	mutex_lock(&group->lock);
476 	list_del(&region->list);
477 	group->region_count--;
478 	mlxsw_sp_acl_tcam_group_update(mlxsw_sp, group);
479 	mutex_unlock(&group->lock);
480 }
481 
482 static int
483 mlxsw_sp_acl_tcam_vgroup_vregion_attach(struct mlxsw_sp *mlxsw_sp,
484 					struct mlxsw_sp_acl_tcam_vgroup *vgroup,
485 					struct mlxsw_sp_acl_tcam_vregion *vregion,
486 					unsigned int priority)
487 {
488 	struct mlxsw_sp_acl_tcam_vregion *vregion2;
489 	struct list_head *pos;
490 	int err;
491 
492 	/* Position the vregion inside the list according to priority */
493 	list_for_each(pos, &vgroup->vregion_list) {
494 		vregion2 = list_entry(pos, typeof(*vregion2), list);
495 		if (mlxsw_sp_acl_tcam_vregion_prio(vregion2) > priority)
496 			break;
497 	}
498 	list_add_tail(&vregion->list, pos);
499 
500 	err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp, &vgroup->group,
501 						    vregion->region,
502 						    priority, NULL);
503 	if (err)
504 		goto err_region_attach;
505 
506 	return 0;
507 
508 err_region_attach:
509 	list_del(&vregion->list);
510 	return err;
511 }
512 
513 static void
514 mlxsw_sp_acl_tcam_vgroup_vregion_detach(struct mlxsw_sp *mlxsw_sp,
515 					struct mlxsw_sp_acl_tcam_vregion *vregion)
516 {
517 	list_del(&vregion->list);
518 	if (vregion->region2)
519 		mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp,
520 						      vregion->region2);
521 	mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, vregion->region);
522 }
523 
524 static struct mlxsw_sp_acl_tcam_vregion *
525 mlxsw_sp_acl_tcam_vgroup_vregion_find(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
526 				      unsigned int priority,
527 				      struct mlxsw_afk_element_usage *elusage,
528 				      bool *p_need_split)
529 {
530 	struct mlxsw_sp_acl_tcam_vregion *vregion, *vregion2;
531 	struct list_head *pos;
532 	bool issubset;
533 
534 	list_for_each(pos, &vgroup->vregion_list) {
535 		vregion = list_entry(pos, typeof(*vregion), list);
536 
537 		/* First, check if the requested priority does not rather belong
538 		 * under some of the next vregions.
539 		 */
540 		if (pos->next != &vgroup->vregion_list) { /* not last */
541 			vregion2 = list_entry(pos->next, typeof(*vregion2),
542 					      list);
543 			if (priority >=
544 			    mlxsw_sp_acl_tcam_vregion_prio(vregion2))
545 				continue;
546 		}
547 
548 		issubset = mlxsw_afk_key_info_subset(vregion->key_info,
549 						     elusage);
550 
551 		/* If requested element usage would not fit and the priority
552 		 * is lower than the currently inspected vregion we cannot
553 		 * use this region, so return NULL to indicate new vregion has
554 		 * to be created.
555 		 */
556 		if (!issubset &&
557 		    priority < mlxsw_sp_acl_tcam_vregion_prio(vregion))
558 			return NULL;
559 
560 		/* If requested element usage would not fit and the priority
561 		 * is higher than the currently inspected vregion we cannot
562 		 * use this vregion. There is still some hope that the next
563 		 * vregion would be the fit. So let it be processed and
564 		 * eventually break at the check right above this.
565 		 */
566 		if (!issubset &&
567 		    priority > mlxsw_sp_acl_tcam_vregion_max_prio(vregion))
568 			continue;
569 
570 		/* Indicate if the vregion needs to be split in order to add
571 		 * the requested priority. Split is needed when requested
572 		 * element usage won't fit into the found vregion.
573 		 */
574 		*p_need_split = !issubset;
575 		return vregion;
576 	}
577 	return NULL; /* New vregion has to be created. */
578 }
579 
580 static void
581 mlxsw_sp_acl_tcam_vgroup_use_patterns(struct mlxsw_sp_acl_tcam_vgroup *vgroup,
582 				      struct mlxsw_afk_element_usage *elusage,
583 				      struct mlxsw_afk_element_usage *out)
584 {
585 	const struct mlxsw_sp_acl_tcam_pattern *pattern;
586 	int i;
587 
588 	/* In case the template is set, we don't have to look up the pattern
589 	 * and just use the template.
590 	 */
591 	if (vgroup->tmplt_elusage_set) {
592 		memcpy(out, &vgroup->tmplt_elusage, sizeof(*out));
593 		WARN_ON(!mlxsw_afk_element_usage_subset(elusage, out));
594 		return;
595 	}
596 
597 	for (i = 0; i < vgroup->patterns_count; i++) {
598 		pattern = &vgroup->patterns[i];
599 		mlxsw_afk_element_usage_fill(out, pattern->elements,
600 					     pattern->elements_count);
601 		if (mlxsw_afk_element_usage_subset(elusage, out))
602 			return;
603 	}
604 	memcpy(out, elusage, sizeof(*out));
605 }
606 
607 static int
608 mlxsw_sp_acl_tcam_region_alloc(struct mlxsw_sp *mlxsw_sp,
609 			       struct mlxsw_sp_acl_tcam_region *region)
610 {
611 	struct mlxsw_afk_key_info *key_info = region->key_info;
612 	char ptar_pl[MLXSW_REG_PTAR_LEN];
613 	unsigned int encodings_count;
614 	int i;
615 	int err;
616 
617 	mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_ALLOC,
618 			    region->key_type,
619 			    MLXSW_SP_ACL_TCAM_REGION_BASE_COUNT,
620 			    region->id, region->tcam_region_info);
621 	encodings_count = mlxsw_afk_key_info_blocks_count_get(key_info);
622 	for (i = 0; i < encodings_count; i++) {
623 		u16 encoding;
624 
625 		encoding = mlxsw_afk_key_info_block_encoding_get(key_info, i);
626 		mlxsw_reg_ptar_key_id_pack(ptar_pl, i, encoding);
627 	}
628 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
629 	if (err)
630 		return err;
631 	mlxsw_reg_ptar_unpack(ptar_pl, region->tcam_region_info);
632 	return 0;
633 }
634 
635 static void
636 mlxsw_sp_acl_tcam_region_free(struct mlxsw_sp *mlxsw_sp,
637 			      struct mlxsw_sp_acl_tcam_region *region)
638 {
639 	char ptar_pl[MLXSW_REG_PTAR_LEN];
640 
641 	mlxsw_reg_ptar_pack(ptar_pl, MLXSW_REG_PTAR_OP_FREE,
642 			    region->key_type, 0, region->id,
643 			    region->tcam_region_info);
644 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptar), ptar_pl);
645 }
646 
647 static int
648 mlxsw_sp_acl_tcam_region_enable(struct mlxsw_sp *mlxsw_sp,
649 				struct mlxsw_sp_acl_tcam_region *region)
650 {
651 	char pacl_pl[MLXSW_REG_PACL_LEN];
652 
653 	mlxsw_reg_pacl_pack(pacl_pl, region->id, true,
654 			    region->tcam_region_info);
655 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
656 }
657 
658 static void
659 mlxsw_sp_acl_tcam_region_disable(struct mlxsw_sp *mlxsw_sp,
660 				 struct mlxsw_sp_acl_tcam_region *region)
661 {
662 	char pacl_pl[MLXSW_REG_PACL_LEN];
663 
664 	mlxsw_reg_pacl_pack(pacl_pl, region->id, false,
665 			    region->tcam_region_info);
666 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pacl), pacl_pl);
667 }
668 
669 static struct mlxsw_sp_acl_tcam_region *
670 mlxsw_sp_acl_tcam_region_create(struct mlxsw_sp *mlxsw_sp,
671 				struct mlxsw_sp_acl_tcam *tcam,
672 				struct mlxsw_sp_acl_tcam_vregion *vregion,
673 				void *hints_priv)
674 {
675 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
676 	struct mlxsw_sp_acl_tcam_region *region;
677 	int err;
678 
679 	region = kzalloc(sizeof(*region) + ops->region_priv_size, GFP_KERNEL);
680 	if (!region)
681 		return ERR_PTR(-ENOMEM);
682 	region->mlxsw_sp = mlxsw_sp;
683 	region->vregion = vregion;
684 	region->key_info = vregion->key_info;
685 
686 	err = mlxsw_sp_acl_tcam_region_id_get(tcam, &region->id);
687 	if (err)
688 		goto err_region_id_get;
689 
690 	err = ops->region_associate(mlxsw_sp, region);
691 	if (err)
692 		goto err_tcam_region_associate;
693 
694 	region->key_type = ops->key_type;
695 	err = mlxsw_sp_acl_tcam_region_alloc(mlxsw_sp, region);
696 	if (err)
697 		goto err_tcam_region_alloc;
698 
699 	err = mlxsw_sp_acl_tcam_region_enable(mlxsw_sp, region);
700 	if (err)
701 		goto err_tcam_region_enable;
702 
703 	err = ops->region_init(mlxsw_sp, region->priv, tcam->priv,
704 			       region, hints_priv);
705 	if (err)
706 		goto err_tcam_region_init;
707 
708 	return region;
709 
710 err_tcam_region_init:
711 	mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
712 err_tcam_region_enable:
713 	mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
714 err_tcam_region_alloc:
715 err_tcam_region_associate:
716 	mlxsw_sp_acl_tcam_region_id_put(tcam, region->id);
717 err_region_id_get:
718 	kfree(region);
719 	return ERR_PTR(err);
720 }
721 
722 static void
723 mlxsw_sp_acl_tcam_region_destroy(struct mlxsw_sp *mlxsw_sp,
724 				 struct mlxsw_sp_acl_tcam_region *region)
725 {
726 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
727 
728 	ops->region_fini(mlxsw_sp, region->priv);
729 	mlxsw_sp_acl_tcam_region_disable(mlxsw_sp, region);
730 	mlxsw_sp_acl_tcam_region_free(mlxsw_sp, region);
731 	mlxsw_sp_acl_tcam_region_id_put(region->group->tcam,
732 					region->id);
733 	kfree(region);
734 }
735 
736 static void
737 mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(struct mlxsw_sp_acl_tcam_vregion *vregion)
738 {
739 	unsigned long interval = vregion->tcam->vregion_rehash_intrvl;
740 
741 	if (!interval)
742 		return;
743 	mlxsw_core_schedule_dw(&vregion->rehash.dw,
744 			       msecs_to_jiffies(interval));
745 }
746 
747 static void
748 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
749 				 struct mlxsw_sp_acl_tcam_vregion *vregion,
750 				 int *credits);
751 
752 static void mlxsw_sp_acl_tcam_vregion_rehash_work(struct work_struct *work)
753 {
754 	struct mlxsw_sp_acl_tcam_vregion *vregion =
755 		container_of(work, struct mlxsw_sp_acl_tcam_vregion,
756 			     rehash.dw.work);
757 	int credits = MLXSW_SP_ACL_TCAM_VREGION_REHASH_CREDITS;
758 
759 	mlxsw_sp_acl_tcam_vregion_rehash(vregion->mlxsw_sp, vregion, &credits);
760 	if (credits < 0)
761 		/* Rehash gone out of credits so it was interrupted.
762 		 * Schedule the work as soon as possible to continue.
763 		 */
764 		mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
765 	else
766 		mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
767 }
768 
769 static void
770 mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(struct mlxsw_sp_acl_tcam_vchunk *vchunk)
771 {
772 	struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
773 
774 	/* If a rule was added or deleted from vchunk which is currently
775 	 * under rehash migration, we have to reset the ventry pointers
776 	 * to make sure all rules are properly migrated.
777 	 */
778 	if (vregion->rehash.ctx.current_vchunk == vchunk) {
779 		vregion->rehash.ctx.start_ventry = NULL;
780 		vregion->rehash.ctx.stop_ventry = NULL;
781 	}
782 }
783 
784 static void
785 mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(struct mlxsw_sp_acl_tcam_vregion *vregion)
786 {
787 	/* If a chunk was added or deleted from vregion we have to reset
788 	 * the current chunk pointer to make sure all chunks
789 	 * are properly migrated.
790 	 */
791 	vregion->rehash.ctx.current_vchunk = NULL;
792 }
793 
794 static struct mlxsw_sp_acl_tcam_vregion *
795 mlxsw_sp_acl_tcam_vregion_create(struct mlxsw_sp *mlxsw_sp,
796 				 struct mlxsw_sp_acl_tcam_vgroup *vgroup,
797 				 unsigned int priority,
798 				 struct mlxsw_afk_element_usage *elusage)
799 {
800 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
801 	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
802 	struct mlxsw_sp_acl_tcam *tcam = vgroup->group.tcam;
803 	struct mlxsw_sp_acl_tcam_vregion *vregion;
804 	int err;
805 
806 	vregion = kzalloc(sizeof(*vregion), GFP_KERNEL);
807 	if (!vregion)
808 		return ERR_PTR(-ENOMEM);
809 	INIT_LIST_HEAD(&vregion->vchunk_list);
810 	mutex_init(&vregion->lock);
811 	vregion->tcam = tcam;
812 	vregion->mlxsw_sp = mlxsw_sp;
813 	vregion->vgroup = vgroup;
814 	vregion->ref_count = 1;
815 
816 	vregion->key_info = mlxsw_afk_key_info_get(afk, elusage);
817 	if (IS_ERR(vregion->key_info)) {
818 		err = PTR_ERR(vregion->key_info);
819 		goto err_key_info_get;
820 	}
821 
822 	vregion->region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, tcam,
823 							  vregion, NULL);
824 	if (IS_ERR(vregion->region)) {
825 		err = PTR_ERR(vregion->region);
826 		goto err_region_create;
827 	}
828 
829 	err = mlxsw_sp_acl_tcam_vgroup_vregion_attach(mlxsw_sp, vgroup, vregion,
830 						      priority);
831 	if (err)
832 		goto err_vgroup_vregion_attach;
833 
834 	if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
835 		/* Create the delayed work for vregion periodic rehash */
836 		INIT_DELAYED_WORK(&vregion->rehash.dw,
837 				  mlxsw_sp_acl_tcam_vregion_rehash_work);
838 		mlxsw_sp_acl_tcam_vregion_rehash_work_schedule(vregion);
839 		mutex_lock(&tcam->lock);
840 		list_add_tail(&vregion->tlist, &tcam->vregion_list);
841 		mutex_unlock(&tcam->lock);
842 	}
843 
844 	return vregion;
845 
846 err_vgroup_vregion_attach:
847 	mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
848 err_region_create:
849 	mlxsw_afk_key_info_put(vregion->key_info);
850 err_key_info_get:
851 	kfree(vregion);
852 	return ERR_PTR(err);
853 }
854 
855 static void
856 mlxsw_sp_acl_tcam_vregion_destroy(struct mlxsw_sp *mlxsw_sp,
857 				  struct mlxsw_sp_acl_tcam_vregion *vregion)
858 {
859 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
860 	struct mlxsw_sp_acl_tcam_vgroup *vgroup = vregion->vgroup;
861 	struct mlxsw_sp_acl_tcam *tcam = vregion->tcam;
862 
863 	if (vgroup->vregion_rehash_enabled && ops->region_rehash_hints_get) {
864 		mutex_lock(&tcam->lock);
865 		list_del(&vregion->tlist);
866 		mutex_unlock(&tcam->lock);
867 		cancel_delayed_work_sync(&vregion->rehash.dw);
868 	}
869 	mlxsw_sp_acl_tcam_vgroup_vregion_detach(mlxsw_sp, vregion);
870 	if (vregion->region2)
871 		mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region2);
872 	mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, vregion->region);
873 	mlxsw_afk_key_info_put(vregion->key_info);
874 	mutex_destroy(&vregion->lock);
875 	kfree(vregion);
876 }
877 
878 u32 mlxsw_sp_acl_tcam_vregion_rehash_intrvl_get(struct mlxsw_sp *mlxsw_sp,
879 						struct mlxsw_sp_acl_tcam *tcam)
880 {
881 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
882 	u32 vregion_rehash_intrvl;
883 
884 	if (WARN_ON(!ops->region_rehash_hints_get))
885 		return 0;
886 	vregion_rehash_intrvl = tcam->vregion_rehash_intrvl;
887 	return vregion_rehash_intrvl;
888 }
889 
890 int mlxsw_sp_acl_tcam_vregion_rehash_intrvl_set(struct mlxsw_sp *mlxsw_sp,
891 						struct mlxsw_sp_acl_tcam *tcam,
892 						u32 val)
893 {
894 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
895 	struct mlxsw_sp_acl_tcam_vregion *vregion;
896 
897 	if (val < MLXSW_SP_ACL_TCAM_VREGION_REHASH_INTRVL_MIN && val)
898 		return -EINVAL;
899 	if (WARN_ON(!ops->region_rehash_hints_get))
900 		return -EOPNOTSUPP;
901 	tcam->vregion_rehash_intrvl = val;
902 	mutex_lock(&tcam->lock);
903 	list_for_each_entry(vregion, &tcam->vregion_list, tlist) {
904 		if (val)
905 			mlxsw_core_schedule_dw(&vregion->rehash.dw, 0);
906 		else
907 			cancel_delayed_work_sync(&vregion->rehash.dw);
908 	}
909 	mutex_unlock(&tcam->lock);
910 	return 0;
911 }
912 
913 static struct mlxsw_sp_acl_tcam_vregion *
914 mlxsw_sp_acl_tcam_vregion_get(struct mlxsw_sp *mlxsw_sp,
915 			      struct mlxsw_sp_acl_tcam_vgroup *vgroup,
916 			      unsigned int priority,
917 			      struct mlxsw_afk_element_usage *elusage)
918 {
919 	struct mlxsw_afk_element_usage vregion_elusage;
920 	struct mlxsw_sp_acl_tcam_vregion *vregion;
921 	bool need_split;
922 
923 	vregion = mlxsw_sp_acl_tcam_vgroup_vregion_find(vgroup, priority,
924 							elusage, &need_split);
925 	if (vregion) {
926 		if (need_split) {
927 			/* According to priority, new vchunk should belong to
928 			 * an existing vregion. However, this vchunk needs
929 			 * elements that vregion does not contain. We need
930 			 * to split the existing vregion into two and create
931 			 * a new vregion for the new vchunk in between.
932 			 * This is not supported now.
933 			 */
934 			return ERR_PTR(-EOPNOTSUPP);
935 		}
936 		vregion->ref_count++;
937 		return vregion;
938 	}
939 
940 	mlxsw_sp_acl_tcam_vgroup_use_patterns(vgroup, elusage,
941 					      &vregion_elusage);
942 
943 	return mlxsw_sp_acl_tcam_vregion_create(mlxsw_sp, vgroup, priority,
944 						&vregion_elusage);
945 }
946 
947 static void
948 mlxsw_sp_acl_tcam_vregion_put(struct mlxsw_sp *mlxsw_sp,
949 			      struct mlxsw_sp_acl_tcam_vregion *vregion)
950 {
951 	if (--vregion->ref_count)
952 		return;
953 	mlxsw_sp_acl_tcam_vregion_destroy(mlxsw_sp, vregion);
954 }
955 
956 static struct mlxsw_sp_acl_tcam_chunk *
957 mlxsw_sp_acl_tcam_chunk_create(struct mlxsw_sp *mlxsw_sp,
958 			       struct mlxsw_sp_acl_tcam_vchunk *vchunk,
959 			       struct mlxsw_sp_acl_tcam_region *region)
960 {
961 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
962 	struct mlxsw_sp_acl_tcam_chunk *chunk;
963 
964 	chunk = kzalloc(sizeof(*chunk) + ops->chunk_priv_size, GFP_KERNEL);
965 	if (!chunk)
966 		return ERR_PTR(-ENOMEM);
967 	chunk->vchunk = vchunk;
968 	chunk->region = region;
969 
970 	ops->chunk_init(region->priv, chunk->priv, vchunk->priority);
971 	return chunk;
972 }
973 
974 static void
975 mlxsw_sp_acl_tcam_chunk_destroy(struct mlxsw_sp *mlxsw_sp,
976 				struct mlxsw_sp_acl_tcam_chunk *chunk)
977 {
978 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
979 
980 	ops->chunk_fini(chunk->priv);
981 	kfree(chunk);
982 }
983 
984 static struct mlxsw_sp_acl_tcam_vchunk *
985 mlxsw_sp_acl_tcam_vchunk_create(struct mlxsw_sp *mlxsw_sp,
986 				struct mlxsw_sp_acl_tcam_vgroup *vgroup,
987 				unsigned int priority,
988 				struct mlxsw_afk_element_usage *elusage)
989 {
990 	struct mlxsw_sp_acl_tcam_vregion *vregion;
991 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
992 	int err;
993 
994 	if (priority == MLXSW_SP_ACL_TCAM_CATCHALL_PRIO)
995 		return ERR_PTR(-EINVAL);
996 
997 	vchunk = kzalloc(sizeof(*vchunk), GFP_KERNEL);
998 	if (!vchunk)
999 		return ERR_PTR(-ENOMEM);
1000 	INIT_LIST_HEAD(&vchunk->ventry_list);
1001 	vchunk->priority = priority;
1002 	vchunk->vgroup = vgroup;
1003 	vchunk->ref_count = 1;
1004 
1005 	vregion = mlxsw_sp_acl_tcam_vregion_get(mlxsw_sp, vgroup,
1006 						priority, elusage);
1007 	if (IS_ERR(vregion)) {
1008 		err = PTR_ERR(vregion);
1009 		goto err_vregion_get;
1010 	}
1011 
1012 	vchunk->vregion = vregion;
1013 
1014 	err = rhashtable_insert_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1015 				     mlxsw_sp_acl_tcam_vchunk_ht_params);
1016 	if (err)
1017 		goto err_rhashtable_insert;
1018 
1019 	mutex_lock(&vregion->lock);
1020 	vchunk->chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk,
1021 						       vchunk->vregion->region);
1022 	if (IS_ERR(vchunk->chunk)) {
1023 		mutex_unlock(&vregion->lock);
1024 		err = PTR_ERR(vchunk->chunk);
1025 		goto err_chunk_create;
1026 	}
1027 
1028 	mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
1029 	list_add_tail(&vchunk->list, &vregion->vchunk_list);
1030 	mutex_unlock(&vregion->lock);
1031 
1032 	return vchunk;
1033 
1034 err_chunk_create:
1035 	rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1036 			       mlxsw_sp_acl_tcam_vchunk_ht_params);
1037 err_rhashtable_insert:
1038 	mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vregion);
1039 err_vregion_get:
1040 	kfree(vchunk);
1041 	return ERR_PTR(err);
1042 }
1043 
1044 static void
1045 mlxsw_sp_acl_tcam_vchunk_destroy(struct mlxsw_sp *mlxsw_sp,
1046 				 struct mlxsw_sp_acl_tcam_vchunk *vchunk)
1047 {
1048 	struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
1049 	struct mlxsw_sp_acl_tcam_vgroup *vgroup = vchunk->vgroup;
1050 
1051 	mutex_lock(&vregion->lock);
1052 	mlxsw_sp_acl_tcam_rehash_ctx_vregion_changed(vregion);
1053 	list_del(&vchunk->list);
1054 	if (vchunk->chunk2)
1055 		mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
1056 	mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk);
1057 	mutex_unlock(&vregion->lock);
1058 	rhashtable_remove_fast(&vgroup->vchunk_ht, &vchunk->ht_node,
1059 			       mlxsw_sp_acl_tcam_vchunk_ht_params);
1060 	mlxsw_sp_acl_tcam_vregion_put(mlxsw_sp, vchunk->vregion);
1061 	kfree(vchunk);
1062 }
1063 
1064 static struct mlxsw_sp_acl_tcam_vchunk *
1065 mlxsw_sp_acl_tcam_vchunk_get(struct mlxsw_sp *mlxsw_sp,
1066 			     struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1067 			     unsigned int priority,
1068 			     struct mlxsw_afk_element_usage *elusage)
1069 {
1070 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1071 
1072 	vchunk = rhashtable_lookup_fast(&vgroup->vchunk_ht, &priority,
1073 					mlxsw_sp_acl_tcam_vchunk_ht_params);
1074 	if (vchunk) {
1075 		if (WARN_ON(!mlxsw_afk_key_info_subset(vchunk->vregion->key_info,
1076 						       elusage)))
1077 			return ERR_PTR(-EINVAL);
1078 		vchunk->ref_count++;
1079 		return vchunk;
1080 	}
1081 	return mlxsw_sp_acl_tcam_vchunk_create(mlxsw_sp, vgroup,
1082 					       priority, elusage);
1083 }
1084 
1085 static void
1086 mlxsw_sp_acl_tcam_vchunk_put(struct mlxsw_sp *mlxsw_sp,
1087 			     struct mlxsw_sp_acl_tcam_vchunk *vchunk)
1088 {
1089 	if (--vchunk->ref_count)
1090 		return;
1091 	mlxsw_sp_acl_tcam_vchunk_destroy(mlxsw_sp, vchunk);
1092 }
1093 
1094 static struct mlxsw_sp_acl_tcam_entry *
1095 mlxsw_sp_acl_tcam_entry_create(struct mlxsw_sp *mlxsw_sp,
1096 			       struct mlxsw_sp_acl_tcam_ventry *ventry,
1097 			       struct mlxsw_sp_acl_tcam_chunk *chunk)
1098 {
1099 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1100 	struct mlxsw_sp_acl_tcam_entry *entry;
1101 	int err;
1102 
1103 	entry = kzalloc(sizeof(*entry) + ops->entry_priv_size, GFP_KERNEL);
1104 	if (!entry)
1105 		return ERR_PTR(-ENOMEM);
1106 	entry->ventry = ventry;
1107 	entry->chunk = chunk;
1108 
1109 	err = ops->entry_add(mlxsw_sp, chunk->region->priv, chunk->priv,
1110 			     entry->priv, ventry->rulei);
1111 	if (err)
1112 		goto err_entry_add;
1113 
1114 	return entry;
1115 
1116 err_entry_add:
1117 	kfree(entry);
1118 	return ERR_PTR(err);
1119 }
1120 
1121 static void mlxsw_sp_acl_tcam_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1122 					    struct mlxsw_sp_acl_tcam_entry *entry)
1123 {
1124 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1125 
1126 	ops->entry_del(mlxsw_sp, entry->chunk->region->priv,
1127 		       entry->chunk->priv, entry->priv);
1128 	kfree(entry);
1129 }
1130 
1131 static int
1132 mlxsw_sp_acl_tcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
1133 				       struct mlxsw_sp_acl_tcam_region *region,
1134 				       struct mlxsw_sp_acl_tcam_entry *entry,
1135 				       struct mlxsw_sp_acl_rule_info *rulei)
1136 {
1137 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1138 
1139 	return ops->entry_action_replace(mlxsw_sp, region->priv,
1140 					 entry->priv, rulei);
1141 }
1142 
1143 static int
1144 mlxsw_sp_acl_tcam_entry_activity_get(struct mlxsw_sp *mlxsw_sp,
1145 				     struct mlxsw_sp_acl_tcam_entry *entry,
1146 				     bool *activity)
1147 {
1148 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1149 
1150 	return ops->entry_activity_get(mlxsw_sp, entry->chunk->region->priv,
1151 				       entry->priv, activity);
1152 }
1153 
1154 static int mlxsw_sp_acl_tcam_ventry_add(struct mlxsw_sp *mlxsw_sp,
1155 					struct mlxsw_sp_acl_tcam_vgroup *vgroup,
1156 					struct mlxsw_sp_acl_tcam_ventry *ventry,
1157 					struct mlxsw_sp_acl_rule_info *rulei)
1158 {
1159 	struct mlxsw_sp_acl_tcam_vregion *vregion;
1160 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1161 	int err;
1162 
1163 	vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp, vgroup, rulei->priority,
1164 					      &rulei->values.elusage);
1165 	if (IS_ERR(vchunk))
1166 		return PTR_ERR(vchunk);
1167 
1168 	ventry->vchunk = vchunk;
1169 	ventry->rulei = rulei;
1170 	vregion = vchunk->vregion;
1171 
1172 	mutex_lock(&vregion->lock);
1173 	ventry->entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry,
1174 						       vchunk->chunk);
1175 	if (IS_ERR(ventry->entry)) {
1176 		mutex_unlock(&vregion->lock);
1177 		err = PTR_ERR(ventry->entry);
1178 		goto err_entry_create;
1179 	}
1180 
1181 	list_add_tail(&ventry->list, &vchunk->ventry_list);
1182 	mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1183 	mutex_unlock(&vregion->lock);
1184 
1185 	return 0;
1186 
1187 err_entry_create:
1188 	mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1189 	return err;
1190 }
1191 
1192 static void mlxsw_sp_acl_tcam_ventry_del(struct mlxsw_sp *mlxsw_sp,
1193 					 struct mlxsw_sp_acl_tcam_ventry *ventry)
1194 {
1195 	struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1196 	struct mlxsw_sp_acl_tcam_vregion *vregion = vchunk->vregion;
1197 
1198 	mutex_lock(&vregion->lock);
1199 	mlxsw_sp_acl_tcam_rehash_ctx_vchunk_changed(vchunk);
1200 	list_del(&ventry->list);
1201 	mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1202 	mutex_unlock(&vregion->lock);
1203 	mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, vchunk);
1204 }
1205 
1206 static int
1207 mlxsw_sp_acl_tcam_ventry_action_replace(struct mlxsw_sp *mlxsw_sp,
1208 					struct mlxsw_sp_acl_tcam_ventry *ventry,
1209 					struct mlxsw_sp_acl_rule_info *rulei)
1210 {
1211 	struct mlxsw_sp_acl_tcam_vchunk *vchunk = ventry->vchunk;
1212 
1213 	return mlxsw_sp_acl_tcam_entry_action_replace(mlxsw_sp,
1214 						      vchunk->vregion->region,
1215 						      ventry->entry, rulei);
1216 }
1217 
1218 static int
1219 mlxsw_sp_acl_tcam_ventry_activity_get(struct mlxsw_sp *mlxsw_sp,
1220 				      struct mlxsw_sp_acl_tcam_ventry *ventry,
1221 				      bool *activity)
1222 {
1223 	return mlxsw_sp_acl_tcam_entry_activity_get(mlxsw_sp,
1224 						    ventry->entry, activity);
1225 }
1226 
1227 static int
1228 mlxsw_sp_acl_tcam_ventry_migrate(struct mlxsw_sp *mlxsw_sp,
1229 				 struct mlxsw_sp_acl_tcam_ventry *ventry,
1230 				 struct mlxsw_sp_acl_tcam_chunk *chunk,
1231 				 int *credits)
1232 {
1233 	struct mlxsw_sp_acl_tcam_entry *new_entry;
1234 
1235 	/* First check if the entry is not already where we want it to be. */
1236 	if (ventry->entry->chunk == chunk)
1237 		return 0;
1238 
1239 	if (--(*credits) < 0)
1240 		return 0;
1241 
1242 	new_entry = mlxsw_sp_acl_tcam_entry_create(mlxsw_sp, ventry, chunk);
1243 	if (IS_ERR(new_entry))
1244 		return PTR_ERR(new_entry);
1245 	mlxsw_sp_acl_tcam_entry_destroy(mlxsw_sp, ventry->entry);
1246 	ventry->entry = new_entry;
1247 	return 0;
1248 }
1249 
1250 static int
1251 mlxsw_sp_acl_tcam_vchunk_migrate_start(struct mlxsw_sp *mlxsw_sp,
1252 				       struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1253 				       struct mlxsw_sp_acl_tcam_region *region,
1254 				       struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1255 {
1256 	struct mlxsw_sp_acl_tcam_chunk *new_chunk;
1257 
1258 	new_chunk = mlxsw_sp_acl_tcam_chunk_create(mlxsw_sp, vchunk, region);
1259 	if (IS_ERR(new_chunk)) {
1260 		if (ctx->this_is_rollback)
1261 			vchunk->vregion->failed_rollback = true;
1262 		return PTR_ERR(new_chunk);
1263 	}
1264 	vchunk->chunk2 = vchunk->chunk;
1265 	vchunk->chunk = new_chunk;
1266 	ctx->current_vchunk = vchunk;
1267 	ctx->start_ventry = NULL;
1268 	ctx->stop_ventry = NULL;
1269 	return 0;
1270 }
1271 
1272 static void
1273 mlxsw_sp_acl_tcam_vchunk_migrate_end(struct mlxsw_sp *mlxsw_sp,
1274 				     struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1275 				     struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1276 {
1277 	mlxsw_sp_acl_tcam_chunk_destroy(mlxsw_sp, vchunk->chunk2);
1278 	vchunk->chunk2 = NULL;
1279 	ctx->current_vchunk = NULL;
1280 }
1281 
1282 static int
1283 mlxsw_sp_acl_tcam_vchunk_migrate_one(struct mlxsw_sp *mlxsw_sp,
1284 				     struct mlxsw_sp_acl_tcam_vchunk *vchunk,
1285 				     struct mlxsw_sp_acl_tcam_region *region,
1286 				     struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1287 				     int *credits)
1288 {
1289 	struct mlxsw_sp_acl_tcam_ventry *ventry;
1290 	int err;
1291 
1292 	if (vchunk->chunk->region != region) {
1293 		err = mlxsw_sp_acl_tcam_vchunk_migrate_start(mlxsw_sp, vchunk,
1294 							     region, ctx);
1295 		if (err)
1296 			return err;
1297 	} else if (!vchunk->chunk2) {
1298 		/* The chunk is already as it should be, nothing to do. */
1299 		return 0;
1300 	}
1301 
1302 	/* If the migration got interrupted, we have the ventry to start from
1303 	 * stored in context.
1304 	 */
1305 	if (ctx->start_ventry)
1306 		ventry = ctx->start_ventry;
1307 	else
1308 		ventry = list_first_entry(&vchunk->ventry_list,
1309 					  typeof(*ventry), list);
1310 
1311 	list_for_each_entry_from(ventry, &vchunk->ventry_list, list) {
1312 		/* During rollback, once we reach the ventry that failed
1313 		 * to migrate, we are done.
1314 		 */
1315 		if (ventry == ctx->stop_ventry)
1316 			break;
1317 
1318 		err = mlxsw_sp_acl_tcam_ventry_migrate(mlxsw_sp, ventry,
1319 						       vchunk->chunk, credits);
1320 		if (err) {
1321 			if (ctx->this_is_rollback)
1322 				return err;
1323 			/* Swap the chunk and chunk2 pointers so the follow-up
1324 			 * rollback call will see the original chunk pointer
1325 			 * in vchunk->chunk.
1326 			 */
1327 			swap(vchunk->chunk, vchunk->chunk2);
1328 			/* The rollback has to be done from beginning of the
1329 			 * chunk, that is why we have to null the start_ventry.
1330 			 * However, we know where to stop the rollback,
1331 			 * at the current ventry.
1332 			 */
1333 			ctx->start_ventry = NULL;
1334 			ctx->stop_ventry = ventry;
1335 			return err;
1336 		} else if (*credits < 0) {
1337 			/* We are out of credits, the rest of the ventries
1338 			 * will be migrated later. Save the ventry
1339 			 * which we ended with.
1340 			 */
1341 			ctx->start_ventry = ventry;
1342 			return 0;
1343 		}
1344 	}
1345 
1346 	mlxsw_sp_acl_tcam_vchunk_migrate_end(mlxsw_sp, vchunk, ctx);
1347 	return 0;
1348 }
1349 
1350 static int
1351 mlxsw_sp_acl_tcam_vchunk_migrate_all(struct mlxsw_sp *mlxsw_sp,
1352 				     struct mlxsw_sp_acl_tcam_vregion *vregion,
1353 				     struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1354 				     int *credits)
1355 {
1356 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1357 	int err;
1358 
1359 	/* If the migration got interrupted, we have the vchunk
1360 	 * we are working on stored in context.
1361 	 */
1362 	if (ctx->current_vchunk)
1363 		vchunk = ctx->current_vchunk;
1364 	else
1365 		vchunk = list_first_entry(&vregion->vchunk_list,
1366 					  typeof(*vchunk), list);
1367 
1368 	list_for_each_entry_from(vchunk, &vregion->vchunk_list, list) {
1369 		err = mlxsw_sp_acl_tcam_vchunk_migrate_one(mlxsw_sp, vchunk,
1370 							   vregion->region,
1371 							   ctx, credits);
1372 		if (err || *credits < 0)
1373 			return err;
1374 	}
1375 	return 0;
1376 }
1377 
1378 static int
1379 mlxsw_sp_acl_tcam_vregion_migrate(struct mlxsw_sp *mlxsw_sp,
1380 				  struct mlxsw_sp_acl_tcam_vregion *vregion,
1381 				  struct mlxsw_sp_acl_tcam_rehash_ctx *ctx,
1382 				  int *credits)
1383 {
1384 	int err, err2;
1385 
1386 	trace_mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion);
1387 	mutex_lock(&vregion->lock);
1388 	err = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1389 						   ctx, credits);
1390 	if (err) {
1391 		/* In case migration was not successful, we need to swap
1392 		 * so the original region pointer is assigned again
1393 		 * to vregion->region.
1394 		 */
1395 		swap(vregion->region, vregion->region2);
1396 		ctx->current_vchunk = NULL;
1397 		ctx->this_is_rollback = true;
1398 		err2 = mlxsw_sp_acl_tcam_vchunk_migrate_all(mlxsw_sp, vregion,
1399 							    ctx, credits);
1400 		if (err2)
1401 			vregion->failed_rollback = true;
1402 	}
1403 	mutex_unlock(&vregion->lock);
1404 	trace_mlxsw_sp_acl_tcam_vregion_migrate_end(mlxsw_sp, vregion);
1405 	return err;
1406 }
1407 
1408 static bool
1409 mlxsw_sp_acl_tcam_vregion_rehash_in_progress(const struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1410 {
1411 	return ctx->hints_priv;
1412 }
1413 
1414 static int
1415 mlxsw_sp_acl_tcam_vregion_rehash_start(struct mlxsw_sp *mlxsw_sp,
1416 				       struct mlxsw_sp_acl_tcam_vregion *vregion,
1417 				       struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1418 {
1419 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1420 	unsigned int priority = mlxsw_sp_acl_tcam_vregion_prio(vregion);
1421 	struct mlxsw_sp_acl_tcam_region *new_region;
1422 	void *hints_priv;
1423 	int err;
1424 
1425 	trace_mlxsw_sp_acl_tcam_vregion_rehash(mlxsw_sp, vregion);
1426 	if (vregion->failed_rollback)
1427 		return -EBUSY;
1428 
1429 	hints_priv = ops->region_rehash_hints_get(vregion->region->priv);
1430 	if (IS_ERR(hints_priv))
1431 		return PTR_ERR(hints_priv);
1432 
1433 	new_region = mlxsw_sp_acl_tcam_region_create(mlxsw_sp, vregion->tcam,
1434 						     vregion, hints_priv);
1435 	if (IS_ERR(new_region)) {
1436 		err = PTR_ERR(new_region);
1437 		goto err_region_create;
1438 	}
1439 
1440 	/* vregion->region contains the pointer to the new region
1441 	 * we are going to migrate to.
1442 	 */
1443 	vregion->region2 = vregion->region;
1444 	vregion->region = new_region;
1445 	err = mlxsw_sp_acl_tcam_group_region_attach(mlxsw_sp,
1446 						    vregion->region2->group,
1447 						    new_region, priority,
1448 						    vregion->region2);
1449 	if (err)
1450 		goto err_group_region_attach;
1451 
1452 	ctx->hints_priv = hints_priv;
1453 	ctx->this_is_rollback = false;
1454 
1455 	return 0;
1456 
1457 err_group_region_attach:
1458 	vregion->region = vregion->region2;
1459 	vregion->region2 = NULL;
1460 	mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, new_region);
1461 err_region_create:
1462 	ops->region_rehash_hints_put(hints_priv);
1463 	return err;
1464 }
1465 
1466 static void
1467 mlxsw_sp_acl_tcam_vregion_rehash_end(struct mlxsw_sp *mlxsw_sp,
1468 				     struct mlxsw_sp_acl_tcam_vregion *vregion,
1469 				     struct mlxsw_sp_acl_tcam_rehash_ctx *ctx)
1470 {
1471 	struct mlxsw_sp_acl_tcam_region *unused_region = vregion->region2;
1472 	const struct mlxsw_sp_acl_tcam_ops *ops = mlxsw_sp->acl_tcam_ops;
1473 
1474 	if (!vregion->failed_rollback) {
1475 		vregion->region2 = NULL;
1476 		mlxsw_sp_acl_tcam_group_region_detach(mlxsw_sp, unused_region);
1477 		mlxsw_sp_acl_tcam_region_destroy(mlxsw_sp, unused_region);
1478 	}
1479 	ops->region_rehash_hints_put(ctx->hints_priv);
1480 	ctx->hints_priv = NULL;
1481 }
1482 
1483 static void
1484 mlxsw_sp_acl_tcam_vregion_rehash(struct mlxsw_sp *mlxsw_sp,
1485 				 struct mlxsw_sp_acl_tcam_vregion *vregion,
1486 				 int *credits)
1487 {
1488 	struct mlxsw_sp_acl_tcam_rehash_ctx *ctx = &vregion->rehash.ctx;
1489 	int err;
1490 
1491 	/* Check if the previous rehash work was interrupted
1492 	 * which means we have to continue it now.
1493 	 * If not, start a new rehash.
1494 	 */
1495 	if (!mlxsw_sp_acl_tcam_vregion_rehash_in_progress(ctx)) {
1496 		err = mlxsw_sp_acl_tcam_vregion_rehash_start(mlxsw_sp,
1497 							     vregion, ctx);
1498 		if (err) {
1499 			if (err != -EAGAIN)
1500 				dev_err(mlxsw_sp->bus_info->dev, "Failed get rehash hints\n");
1501 			return;
1502 		}
1503 	}
1504 
1505 	err = mlxsw_sp_acl_tcam_vregion_migrate(mlxsw_sp, vregion,
1506 						ctx, credits);
1507 	if (err) {
1508 		dev_err(mlxsw_sp->bus_info->dev, "Failed to migrate vregion\n");
1509 		if (vregion->failed_rollback) {
1510 			trace_mlxsw_sp_acl_tcam_vregion_rehash_dis(mlxsw_sp,
1511 								   vregion);
1512 			dev_err(mlxsw_sp->bus_info->dev, "Failed to rollback during vregion migration fail\n");
1513 		}
1514 	}
1515 
1516 	if (*credits >= 0)
1517 		mlxsw_sp_acl_tcam_vregion_rehash_end(mlxsw_sp, vregion, ctx);
1518 }
1519 
1520 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv4[] = {
1521 	MLXSW_AFK_ELEMENT_SRC_SYS_PORT,
1522 	MLXSW_AFK_ELEMENT_DMAC_32_47,
1523 	MLXSW_AFK_ELEMENT_DMAC_0_31,
1524 	MLXSW_AFK_ELEMENT_SMAC_32_47,
1525 	MLXSW_AFK_ELEMENT_SMAC_0_31,
1526 	MLXSW_AFK_ELEMENT_ETHERTYPE,
1527 	MLXSW_AFK_ELEMENT_IP_PROTO,
1528 	MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1529 	MLXSW_AFK_ELEMENT_DST_IP_0_31,
1530 	MLXSW_AFK_ELEMENT_DST_L4_PORT,
1531 	MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1532 	MLXSW_AFK_ELEMENT_VID,
1533 	MLXSW_AFK_ELEMENT_PCP,
1534 	MLXSW_AFK_ELEMENT_TCP_FLAGS,
1535 	MLXSW_AFK_ELEMENT_IP_TTL_,
1536 	MLXSW_AFK_ELEMENT_IP_ECN,
1537 	MLXSW_AFK_ELEMENT_IP_DSCP,
1538 };
1539 
1540 static const enum mlxsw_afk_element mlxsw_sp_acl_tcam_pattern_ipv6[] = {
1541 	MLXSW_AFK_ELEMENT_ETHERTYPE,
1542 	MLXSW_AFK_ELEMENT_IP_PROTO,
1543 	MLXSW_AFK_ELEMENT_SRC_IP_96_127,
1544 	MLXSW_AFK_ELEMENT_SRC_IP_64_95,
1545 	MLXSW_AFK_ELEMENT_SRC_IP_32_63,
1546 	MLXSW_AFK_ELEMENT_SRC_IP_0_31,
1547 	MLXSW_AFK_ELEMENT_DST_IP_96_127,
1548 	MLXSW_AFK_ELEMENT_DST_IP_64_95,
1549 	MLXSW_AFK_ELEMENT_DST_IP_32_63,
1550 	MLXSW_AFK_ELEMENT_DST_IP_0_31,
1551 	MLXSW_AFK_ELEMENT_DST_L4_PORT,
1552 	MLXSW_AFK_ELEMENT_SRC_L4_PORT,
1553 };
1554 
1555 static const struct mlxsw_sp_acl_tcam_pattern mlxsw_sp_acl_tcam_patterns[] = {
1556 	{
1557 		.elements = mlxsw_sp_acl_tcam_pattern_ipv4,
1558 		.elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv4),
1559 	},
1560 	{
1561 		.elements = mlxsw_sp_acl_tcam_pattern_ipv6,
1562 		.elements_count = ARRAY_SIZE(mlxsw_sp_acl_tcam_pattern_ipv6),
1563 	},
1564 };
1565 
1566 #define MLXSW_SP_ACL_TCAM_PATTERNS_COUNT \
1567 	ARRAY_SIZE(mlxsw_sp_acl_tcam_patterns)
1568 
1569 struct mlxsw_sp_acl_tcam_flower_ruleset {
1570 	struct mlxsw_sp_acl_tcam_vgroup vgroup;
1571 };
1572 
1573 struct mlxsw_sp_acl_tcam_flower_rule {
1574 	struct mlxsw_sp_acl_tcam_ventry ventry;
1575 };
1576 
1577 static int
1578 mlxsw_sp_acl_tcam_flower_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1579 				     struct mlxsw_sp_acl_tcam *tcam,
1580 				     void *ruleset_priv,
1581 				     struct mlxsw_afk_element_usage *tmplt_elusage)
1582 {
1583 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1584 
1585 	return mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1586 					    mlxsw_sp_acl_tcam_patterns,
1587 					    MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1588 					    tmplt_elusage, true);
1589 }
1590 
1591 static void
1592 mlxsw_sp_acl_tcam_flower_ruleset_del(struct mlxsw_sp *mlxsw_sp,
1593 				     void *ruleset_priv)
1594 {
1595 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1596 
1597 	mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1598 }
1599 
1600 static int
1601 mlxsw_sp_acl_tcam_flower_ruleset_bind(struct mlxsw_sp *mlxsw_sp,
1602 				      void *ruleset_priv,
1603 				      struct mlxsw_sp_port *mlxsw_sp_port,
1604 				      bool ingress)
1605 {
1606 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1607 
1608 	return mlxsw_sp_acl_tcam_group_bind(mlxsw_sp, &ruleset->vgroup.group,
1609 					    mlxsw_sp_port, ingress);
1610 }
1611 
1612 static void
1613 mlxsw_sp_acl_tcam_flower_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1614 					void *ruleset_priv,
1615 					struct mlxsw_sp_port *mlxsw_sp_port,
1616 					bool ingress)
1617 {
1618 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1619 
1620 	mlxsw_sp_acl_tcam_group_unbind(mlxsw_sp, &ruleset->vgroup.group,
1621 				       mlxsw_sp_port, ingress);
1622 }
1623 
1624 static u16
1625 mlxsw_sp_acl_tcam_flower_ruleset_group_id(void *ruleset_priv)
1626 {
1627 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1628 
1629 	return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1630 }
1631 
1632 static int
1633 mlxsw_sp_acl_tcam_flower_rule_add(struct mlxsw_sp *mlxsw_sp,
1634 				  void *ruleset_priv, void *rule_priv,
1635 				  struct mlxsw_sp_acl_rule_info *rulei)
1636 {
1637 	struct mlxsw_sp_acl_tcam_flower_ruleset *ruleset = ruleset_priv;
1638 	struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1639 
1640 	return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1641 					    &rule->ventry, rulei);
1642 }
1643 
1644 static void
1645 mlxsw_sp_acl_tcam_flower_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1646 {
1647 	struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1648 
1649 	mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1650 }
1651 
1652 static int
1653 mlxsw_sp_acl_tcam_flower_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1654 					     void *rule_priv,
1655 					     struct mlxsw_sp_acl_rule_info *rulei)
1656 {
1657 	return -EOPNOTSUPP;
1658 }
1659 
1660 static int
1661 mlxsw_sp_acl_tcam_flower_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1662 					   void *rule_priv, bool *activity)
1663 {
1664 	struct mlxsw_sp_acl_tcam_flower_rule *rule = rule_priv;
1665 
1666 	return mlxsw_sp_acl_tcam_ventry_activity_get(mlxsw_sp, &rule->ventry,
1667 						     activity);
1668 }
1669 
1670 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_flower_ops = {
1671 	.ruleset_priv_size	= sizeof(struct mlxsw_sp_acl_tcam_flower_ruleset),
1672 	.ruleset_add		= mlxsw_sp_acl_tcam_flower_ruleset_add,
1673 	.ruleset_del		= mlxsw_sp_acl_tcam_flower_ruleset_del,
1674 	.ruleset_bind		= mlxsw_sp_acl_tcam_flower_ruleset_bind,
1675 	.ruleset_unbind		= mlxsw_sp_acl_tcam_flower_ruleset_unbind,
1676 	.ruleset_group_id	= mlxsw_sp_acl_tcam_flower_ruleset_group_id,
1677 	.rule_priv_size		= sizeof(struct mlxsw_sp_acl_tcam_flower_rule),
1678 	.rule_add		= mlxsw_sp_acl_tcam_flower_rule_add,
1679 	.rule_del		= mlxsw_sp_acl_tcam_flower_rule_del,
1680 	.rule_action_replace	= mlxsw_sp_acl_tcam_flower_rule_action_replace,
1681 	.rule_activity_get	= mlxsw_sp_acl_tcam_flower_rule_activity_get,
1682 };
1683 
1684 struct mlxsw_sp_acl_tcam_mr_ruleset {
1685 	struct mlxsw_sp_acl_tcam_vchunk *vchunk;
1686 	struct mlxsw_sp_acl_tcam_vgroup vgroup;
1687 };
1688 
1689 struct mlxsw_sp_acl_tcam_mr_rule {
1690 	struct mlxsw_sp_acl_tcam_ventry ventry;
1691 };
1692 
1693 static int
1694 mlxsw_sp_acl_tcam_mr_ruleset_add(struct mlxsw_sp *mlxsw_sp,
1695 				 struct mlxsw_sp_acl_tcam *tcam,
1696 				 void *ruleset_priv,
1697 				 struct mlxsw_afk_element_usage *tmplt_elusage)
1698 {
1699 	struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1700 	int err;
1701 
1702 	err = mlxsw_sp_acl_tcam_vgroup_add(mlxsw_sp, tcam, &ruleset->vgroup,
1703 					   mlxsw_sp_acl_tcam_patterns,
1704 					   MLXSW_SP_ACL_TCAM_PATTERNS_COUNT,
1705 					   tmplt_elusage, false);
1706 	if (err)
1707 		return err;
1708 
1709 	/* For most of the TCAM clients it would make sense to take a tcam chunk
1710 	 * only when the first rule is written. This is not the case for
1711 	 * multicast router as it is required to bind the multicast router to a
1712 	 * specific ACL Group ID which must exist in HW before multicast router
1713 	 * is initialized.
1714 	 */
1715 	ruleset->vchunk = mlxsw_sp_acl_tcam_vchunk_get(mlxsw_sp,
1716 						       &ruleset->vgroup, 1,
1717 						       tmplt_elusage);
1718 	if (IS_ERR(ruleset->vchunk)) {
1719 		err = PTR_ERR(ruleset->vchunk);
1720 		goto err_chunk_get;
1721 	}
1722 
1723 	return 0;
1724 
1725 err_chunk_get:
1726 	mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1727 	return err;
1728 }
1729 
1730 static void
1731 mlxsw_sp_acl_tcam_mr_ruleset_del(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv)
1732 {
1733 	struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1734 
1735 	mlxsw_sp_acl_tcam_vchunk_put(mlxsw_sp, ruleset->vchunk);
1736 	mlxsw_sp_acl_tcam_vgroup_del(&ruleset->vgroup);
1737 }
1738 
1739 static int
1740 mlxsw_sp_acl_tcam_mr_ruleset_bind(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1741 				  struct mlxsw_sp_port *mlxsw_sp_port,
1742 				  bool ingress)
1743 {
1744 	/* Binding is done when initializing multicast router */
1745 	return 0;
1746 }
1747 
1748 static void
1749 mlxsw_sp_acl_tcam_mr_ruleset_unbind(struct mlxsw_sp *mlxsw_sp,
1750 				    void *ruleset_priv,
1751 				    struct mlxsw_sp_port *mlxsw_sp_port,
1752 				    bool ingress)
1753 {
1754 }
1755 
1756 static u16
1757 mlxsw_sp_acl_tcam_mr_ruleset_group_id(void *ruleset_priv)
1758 {
1759 	struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1760 
1761 	return mlxsw_sp_acl_tcam_group_id(&ruleset->vgroup.group);
1762 }
1763 
1764 static int
1765 mlxsw_sp_acl_tcam_mr_rule_add(struct mlxsw_sp *mlxsw_sp, void *ruleset_priv,
1766 			      void *rule_priv,
1767 			      struct mlxsw_sp_acl_rule_info *rulei)
1768 {
1769 	struct mlxsw_sp_acl_tcam_mr_ruleset *ruleset = ruleset_priv;
1770 	struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1771 
1772 	return mlxsw_sp_acl_tcam_ventry_add(mlxsw_sp, &ruleset->vgroup,
1773 					   &rule->ventry, rulei);
1774 }
1775 
1776 static void
1777 mlxsw_sp_acl_tcam_mr_rule_del(struct mlxsw_sp *mlxsw_sp, void *rule_priv)
1778 {
1779 	struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1780 
1781 	mlxsw_sp_acl_tcam_ventry_del(mlxsw_sp, &rule->ventry);
1782 }
1783 
1784 static int
1785 mlxsw_sp_acl_tcam_mr_rule_action_replace(struct mlxsw_sp *mlxsw_sp,
1786 					 void *rule_priv,
1787 					 struct mlxsw_sp_acl_rule_info *rulei)
1788 {
1789 	struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1790 
1791 	return mlxsw_sp_acl_tcam_ventry_action_replace(mlxsw_sp, &rule->ventry,
1792 						       rulei);
1793 }
1794 
1795 static int
1796 mlxsw_sp_acl_tcam_mr_rule_activity_get(struct mlxsw_sp *mlxsw_sp,
1797 				       void *rule_priv, bool *activity)
1798 {
1799 	struct mlxsw_sp_acl_tcam_mr_rule *rule = rule_priv;
1800 
1801 	return mlxsw_sp_acl_tcam_ventry_activity_get(mlxsw_sp, &rule->ventry,
1802 						     activity);
1803 }
1804 
1805 static const struct mlxsw_sp_acl_profile_ops mlxsw_sp_acl_tcam_mr_ops = {
1806 	.ruleset_priv_size	= sizeof(struct mlxsw_sp_acl_tcam_mr_ruleset),
1807 	.ruleset_add		= mlxsw_sp_acl_tcam_mr_ruleset_add,
1808 	.ruleset_del		= mlxsw_sp_acl_tcam_mr_ruleset_del,
1809 	.ruleset_bind		= mlxsw_sp_acl_tcam_mr_ruleset_bind,
1810 	.ruleset_unbind		= mlxsw_sp_acl_tcam_mr_ruleset_unbind,
1811 	.ruleset_group_id	= mlxsw_sp_acl_tcam_mr_ruleset_group_id,
1812 	.rule_priv_size		= sizeof(struct mlxsw_sp_acl_tcam_mr_rule),
1813 	.rule_add		= mlxsw_sp_acl_tcam_mr_rule_add,
1814 	.rule_del		= mlxsw_sp_acl_tcam_mr_rule_del,
1815 	.rule_action_replace	= mlxsw_sp_acl_tcam_mr_rule_action_replace,
1816 	.rule_activity_get	= mlxsw_sp_acl_tcam_mr_rule_activity_get,
1817 };
1818 
1819 static const struct mlxsw_sp_acl_profile_ops *
1820 mlxsw_sp_acl_tcam_profile_ops_arr[] = {
1821 	[MLXSW_SP_ACL_PROFILE_FLOWER] = &mlxsw_sp_acl_tcam_flower_ops,
1822 	[MLXSW_SP_ACL_PROFILE_MR] = &mlxsw_sp_acl_tcam_mr_ops,
1823 };
1824 
1825 const struct mlxsw_sp_acl_profile_ops *
1826 mlxsw_sp_acl_tcam_profile_ops(struct mlxsw_sp *mlxsw_sp,
1827 			      enum mlxsw_sp_acl_profile profile)
1828 {
1829 	const struct mlxsw_sp_acl_profile_ops *ops;
1830 
1831 	if (WARN_ON(profile >= ARRAY_SIZE(mlxsw_sp_acl_tcam_profile_ops_arr)))
1832 		return NULL;
1833 	ops = mlxsw_sp_acl_tcam_profile_ops_arr[profile];
1834 	if (WARN_ON(!ops))
1835 		return NULL;
1836 	return ops;
1837 }
1838