1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/err.h>
6 #include <linux/errno.h>
7 #include <linux/gfp.h>
8 #include <linux/refcount.h>
9 #include <linux/rhashtable.h>
10 #define CREATE_TRACE_POINTS
11 #include <trace/events/mlxsw.h>
12 
13 #include "reg.h"
14 #include "core.h"
15 #include "spectrum.h"
16 #include "spectrum_acl_tcam.h"
17 #include "core_acl_flex_keys.h"
18 
19 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START	0
20 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END	5
21 
22 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
23 	char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
24 	u8 erp_id;
25 };
26 
27 struct mlxsw_sp_acl_atcam_lkey_id {
28 	struct rhash_head ht_node;
29 	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
30 	refcount_t refcnt;
31 	u32 id;
32 };
33 
34 struct mlxsw_sp_acl_atcam_region_ops {
35 	int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
36 	void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
37 	struct mlxsw_sp_acl_atcam_lkey_id *
38 		(*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
39 			       char *enc_key, u8 erp_id);
40 	void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
41 			    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
42 };
43 
44 struct mlxsw_sp_acl_atcam_region_generic {
45 	struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
46 };
47 
48 struct mlxsw_sp_acl_atcam_region_12kb {
49 	struct rhashtable lkey_ht;
50 	unsigned int max_lkey_id;
51 	unsigned long *used_lkey_id;
52 };
53 
54 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
55 	.key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
56 	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
57 	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
58 };
59 
60 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
61 	.key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
62 	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
63 	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
64 };
65 
66 static bool
67 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
68 {
69 	return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
70 }
71 
72 static int
73 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
74 {
75 	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
76 
77 	region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
78 	if (!region_generic)
79 		return -ENOMEM;
80 
81 	refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
82 	aregion->priv = region_generic;
83 
84 	return 0;
85 }
86 
87 static void
88 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
89 {
90 	kfree(aregion->priv);
91 }
92 
93 static struct mlxsw_sp_acl_atcam_lkey_id *
94 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
95 				       char *enc_key, u8 erp_id)
96 {
97 	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
98 
99 	region_generic = aregion->priv;
100 	return &region_generic->dummy_lkey_id;
101 }
102 
103 static void
104 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
105 				       struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
106 {
107 }
108 
109 static const struct mlxsw_sp_acl_atcam_region_ops
110 mlxsw_sp_acl_atcam_region_generic_ops = {
111 	.init		= mlxsw_sp_acl_atcam_region_generic_init,
112 	.fini		= mlxsw_sp_acl_atcam_region_generic_fini,
113 	.lkey_id_get	= mlxsw_sp_acl_atcam_generic_lkey_id_get,
114 	.lkey_id_put	= mlxsw_sp_acl_atcam_generic_lkey_id_put,
115 };
116 
117 static int
118 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
119 {
120 	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
121 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
122 	u64 max_lkey_id;
123 	int err;
124 
125 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
126 		return -EIO;
127 
128 	max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
129 	region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
130 	if (!region_12kb)
131 		return -ENOMEM;
132 
133 	region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL);
134 	if (!region_12kb->used_lkey_id) {
135 		err = -ENOMEM;
136 		goto err_used_lkey_id_alloc;
137 	}
138 
139 	err = rhashtable_init(&region_12kb->lkey_ht,
140 			      &mlxsw_sp_acl_atcam_lkey_id_ht_params);
141 	if (err)
142 		goto err_rhashtable_init;
143 
144 	region_12kb->max_lkey_id = max_lkey_id;
145 	aregion->priv = region_12kb;
146 
147 	return 0;
148 
149 err_rhashtable_init:
150 	bitmap_free(region_12kb->used_lkey_id);
151 err_used_lkey_id_alloc:
152 	kfree(region_12kb);
153 	return err;
154 }
155 
156 static void
157 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
158 {
159 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
160 
161 	rhashtable_destroy(&region_12kb->lkey_ht);
162 	bitmap_free(region_12kb->used_lkey_id);
163 	kfree(region_12kb);
164 }
165 
166 static struct mlxsw_sp_acl_atcam_lkey_id *
167 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
168 				  struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
169 {
170 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
171 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
172 	u32 id;
173 	int err;
174 
175 	id = find_first_zero_bit(region_12kb->used_lkey_id,
176 				 region_12kb->max_lkey_id);
177 	if (id < region_12kb->max_lkey_id)
178 		__set_bit(id, region_12kb->used_lkey_id);
179 	else
180 		return ERR_PTR(-ENOBUFS);
181 
182 	lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
183 	if (!lkey_id) {
184 		err = -ENOMEM;
185 		goto err_lkey_id_alloc;
186 	}
187 
188 	lkey_id->id = id;
189 	memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
190 	refcount_set(&lkey_id->refcnt, 1);
191 
192 	err = rhashtable_insert_fast(&region_12kb->lkey_ht,
193 				     &lkey_id->ht_node,
194 				     mlxsw_sp_acl_atcam_lkey_id_ht_params);
195 	if (err)
196 		goto err_rhashtable_insert;
197 
198 	return lkey_id;
199 
200 err_rhashtable_insert:
201 	kfree(lkey_id);
202 err_lkey_id_alloc:
203 	__clear_bit(id, region_12kb->used_lkey_id);
204 	return ERR_PTR(err);
205 }
206 
207 static void
208 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
209 				   struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
210 {
211 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
212 	u32 id = lkey_id->id;
213 
214 	rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
215 			       mlxsw_sp_acl_atcam_lkey_id_ht_params);
216 	kfree(lkey_id);
217 	__clear_bit(id, region_12kb->used_lkey_id);
218 }
219 
220 static struct mlxsw_sp_acl_atcam_lkey_id *
221 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
222 				    char *enc_key, u8 erp_id)
223 {
224 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
225 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
226 	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
227 	struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
228 	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
229 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
230 
231 	memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
232 	mlxsw_afk_clear(afk, ht_key.enc_key,
233 			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
234 			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
235 	ht_key.erp_id = erp_id;
236 	lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
237 					 mlxsw_sp_acl_atcam_lkey_id_ht_params);
238 	if (lkey_id) {
239 		refcount_inc(&lkey_id->refcnt);
240 		return lkey_id;
241 	}
242 
243 	return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
244 }
245 
246 static void
247 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
248 				    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
249 {
250 	if (refcount_dec_and_test(&lkey_id->refcnt))
251 		mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
252 }
253 
254 static const struct mlxsw_sp_acl_atcam_region_ops
255 mlxsw_sp_acl_atcam_region_12kb_ops = {
256 	.init		= mlxsw_sp_acl_atcam_region_12kb_init,
257 	.fini		= mlxsw_sp_acl_atcam_region_12kb_fini,
258 	.lkey_id_get	= mlxsw_sp_acl_atcam_12kb_lkey_id_get,
259 	.lkey_id_put	= mlxsw_sp_acl_atcam_12kb_lkey_id_put,
260 };
261 
262 static const struct mlxsw_sp_acl_atcam_region_ops *
263 mlxsw_sp_acl_atcam_region_ops_arr[] = {
264 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]	=
265 		&mlxsw_sp_acl_atcam_region_generic_ops,
266 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]	=
267 		&mlxsw_sp_acl_atcam_region_generic_ops,
268 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]	=
269 		&mlxsw_sp_acl_atcam_region_generic_ops,
270 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]	=
271 		&mlxsw_sp_acl_atcam_region_12kb_ops,
272 };
273 
274 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
275 					u16 region_id)
276 {
277 	char perar_pl[MLXSW_REG_PERAR_LEN];
278 	/* For now, just assume that every region has 12 key blocks */
279 	u16 hw_region = region_id * 3;
280 	u64 max_regions;
281 
282 	max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
283 	if (hw_region >= max_regions)
284 		return -ENOBUFS;
285 
286 	mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
287 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
288 }
289 
290 static void
291 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
292 {
293 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
294 	enum mlxsw_sp_acl_atcam_region_type region_type;
295 	unsigned int blocks_count;
296 
297 	/* We already know the blocks count can not exceed the maximum
298 	 * blocks count.
299 	 */
300 	blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
301 	if (blocks_count <= 2)
302 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
303 	else if (blocks_count <= 4)
304 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
305 	else if (blocks_count <= 8)
306 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
307 	else
308 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
309 
310 	aregion->type = region_type;
311 	aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
312 }
313 
314 int
315 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
316 			       struct mlxsw_sp_acl_atcam *atcam,
317 			       struct mlxsw_sp_acl_atcam_region *aregion,
318 			       struct mlxsw_sp_acl_tcam_region *region,
319 			       void *hints_priv,
320 			       const struct mlxsw_sp_acl_ctcam_region_ops *ops)
321 {
322 	int err;
323 
324 	aregion->region = region;
325 	aregion->atcam = atcam;
326 	mlxsw_sp_acl_atcam_region_type_init(aregion);
327 	INIT_LIST_HEAD(&aregion->entries_list);
328 
329 	err = rhashtable_init(&aregion->entries_ht,
330 			      &mlxsw_sp_acl_atcam_entries_ht_params);
331 	if (err)
332 		return err;
333 	err = aregion->ops->init(aregion);
334 	if (err)
335 		goto err_ops_init;
336 	err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv);
337 	if (err)
338 		goto err_erp_region_init;
339 	err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
340 					     region, ops);
341 	if (err)
342 		goto err_ctcam_region_init;
343 
344 	return 0;
345 
346 err_ctcam_region_init:
347 	mlxsw_sp_acl_erp_region_fini(aregion);
348 err_erp_region_init:
349 	aregion->ops->fini(aregion);
350 err_ops_init:
351 	rhashtable_destroy(&aregion->entries_ht);
352 	return err;
353 }
354 
355 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
356 {
357 	mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
358 	mlxsw_sp_acl_erp_region_fini(aregion);
359 	aregion->ops->fini(aregion);
360 	rhashtable_destroy(&aregion->entries_ht);
361 	WARN_ON(!list_empty(&aregion->entries_list));
362 }
363 
364 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
365 				   struct mlxsw_sp_acl_atcam_chunk *achunk,
366 				   unsigned int priority)
367 {
368 	mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
369 				      priority);
370 }
371 
372 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
373 {
374 	mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
375 }
376 
377 static int
378 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
379 				       struct mlxsw_sp_acl_atcam_region *aregion,
380 				       struct mlxsw_sp_acl_atcam_entry *aentry,
381 				       struct mlxsw_sp_acl_rule_info *rulei)
382 {
383 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
384 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
385 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
386 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
387 	u32 kvdl_index, priority;
388 	int err;
389 
390 	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
391 	if (err)
392 		return err;
393 
394 	lkey_id = aregion->ops->lkey_id_get(aregion, aentry->enc_key, erp_id);
395 	if (IS_ERR(lkey_id))
396 		return PTR_ERR(lkey_id);
397 	aentry->lkey_id = lkey_id;
398 
399 	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
400 	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
401 			     priority, region->tcam_region_info,
402 			     aentry->enc_key, erp_id,
403 			     aentry->delta_info.start,
404 			     aentry->delta_info.mask,
405 			     aentry->delta_info.value,
406 			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
407 			     kvdl_index);
408 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
409 	if (err)
410 		goto err_ptce3_write;
411 
412 	return 0;
413 
414 err_ptce3_write:
415 	aregion->ops->lkey_id_put(aregion, lkey_id);
416 	return err;
417 }
418 
419 static void
420 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
421 				       struct mlxsw_sp_acl_atcam_region *aregion,
422 				       struct mlxsw_sp_acl_atcam_entry *aentry)
423 {
424 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
425 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
426 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
427 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
428 
429 	mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
430 			     region->tcam_region_info,
431 			     aentry->enc_key, erp_id,
432 			     aentry->delta_info.start,
433 			     aentry->delta_info.mask,
434 			     aentry->delta_info.value,
435 			     refcount_read(&lkey_id->refcnt) != 1,
436 			     lkey_id->id, 0);
437 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
438 	aregion->ops->lkey_id_put(aregion, lkey_id);
439 }
440 
441 static int
442 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
443 					       struct mlxsw_sp_acl_atcam_region *aregion,
444 					       struct mlxsw_sp_acl_atcam_entry *aentry,
445 					       struct mlxsw_sp_acl_rule_info *rulei)
446 {
447 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
448 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
449 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
450 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
451 	u32 kvdl_index, priority;
452 	int err;
453 
454 	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
455 	if (err)
456 		return err;
457 	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
458 	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
459 			     priority, region->tcam_region_info,
460 			     aentry->enc_key, erp_id,
461 			     aentry->delta_info.start,
462 			     aentry->delta_info.mask,
463 			     aentry->delta_info.value,
464 			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
465 			     kvdl_index);
466 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
467 }
468 
469 static int
470 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
471 			       struct mlxsw_sp_acl_atcam_region *aregion,
472 			       struct mlxsw_sp_acl_atcam_entry *aentry,
473 			       struct mlxsw_sp_acl_rule_info *rulei)
474 {
475 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
476 	char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
477 	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
478 	const struct mlxsw_sp_acl_erp_delta *delta;
479 	struct mlxsw_sp_acl_erp_mask *erp_mask;
480 	int err;
481 
482 	mlxsw_afk_encode(afk, region->key_info, &rulei->values,
483 			 aentry->ht_key.full_enc_key, mask);
484 
485 	erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
486 	if (IS_ERR(erp_mask))
487 		return PTR_ERR(erp_mask);
488 	aentry->erp_mask = erp_mask;
489 	aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
490 	memcpy(aentry->enc_key, aentry->ht_key.full_enc_key,
491 	       sizeof(aentry->enc_key));
492 
493 	/* Compute all needed delta information and clear the delta bits
494 	 * from the encrypted key.
495 	 */
496 	delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
497 	aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
498 	aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
499 	aentry->delta_info.value =
500 		mlxsw_sp_acl_erp_delta_value(delta,
501 					     aentry->ht_key.full_enc_key);
502 	mlxsw_sp_acl_erp_delta_clear(delta, aentry->enc_key);
503 
504 	/* Add rule to the list of A-TCAM rules, assuming this
505 	 * rule is intended to A-TCAM. In case this rule does
506 	 * not fit into A-TCAM it will be removed from the list.
507 	 */
508 	list_add(&aentry->list, &aregion->entries_list);
509 
510 	/* We can't insert identical rules into the A-TCAM, so fail and
511 	 * let the rule spill into C-TCAM
512 	 */
513 	err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
514 					    &aentry->ht_node,
515 					    mlxsw_sp_acl_atcam_entries_ht_params);
516 	if (err)
517 		goto err_rhashtable_insert;
518 
519 	/* Bloom filter must be updated here, before inserting the rule into
520 	 * the A-TCAM.
521 	 */
522 	err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
523 	if (err)
524 		goto err_bf_insert;
525 
526 	err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
527 						     rulei);
528 	if (err)
529 		goto err_rule_insert;
530 
531 	return 0;
532 
533 err_rule_insert:
534 	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
535 err_bf_insert:
536 	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
537 			       mlxsw_sp_acl_atcam_entries_ht_params);
538 err_rhashtable_insert:
539 	list_del(&aentry->list);
540 	mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
541 	return err;
542 }
543 
544 static void
545 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
546 			       struct mlxsw_sp_acl_atcam_region *aregion,
547 			       struct mlxsw_sp_acl_atcam_entry *aentry)
548 {
549 	mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
550 	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
551 	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
552 			       mlxsw_sp_acl_atcam_entries_ht_params);
553 	list_del(&aentry->list);
554 	mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
555 }
556 
557 static int
558 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
559 					  struct mlxsw_sp_acl_atcam_region *aregion,
560 					  struct mlxsw_sp_acl_atcam_entry *aentry,
561 					  struct mlxsw_sp_acl_rule_info *rulei)
562 {
563 	return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
564 							      aentry, rulei);
565 }
566 
567 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
568 				 struct mlxsw_sp_acl_atcam_region *aregion,
569 				 struct mlxsw_sp_acl_atcam_chunk *achunk,
570 				 struct mlxsw_sp_acl_atcam_entry *aentry,
571 				 struct mlxsw_sp_acl_rule_info *rulei)
572 {
573 	int err;
574 
575 	err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
576 	if (!err)
577 		return 0;
578 
579 	/* It is possible we failed to add the rule to the A-TCAM due to
580 	 * exceeded number of masks. Try to spill into C-TCAM.
581 	 */
582 	trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion);
583 	err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
584 					   &achunk->cchunk, &aentry->centry,
585 					   rulei, true);
586 	if (!err)
587 		return 0;
588 
589 	return err;
590 }
591 
592 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
593 				  struct mlxsw_sp_acl_atcam_region *aregion,
594 				  struct mlxsw_sp_acl_atcam_chunk *achunk,
595 				  struct mlxsw_sp_acl_atcam_entry *aentry)
596 {
597 	if (mlxsw_sp_acl_atcam_is_centry(aentry))
598 		mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
599 					     &achunk->cchunk, &aentry->centry);
600 	else
601 		__mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
602 }
603 
604 int
605 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
606 					struct mlxsw_sp_acl_atcam_region *aregion,
607 					struct mlxsw_sp_acl_atcam_entry *aentry,
608 					struct mlxsw_sp_acl_rule_info *rulei)
609 {
610 	int err;
611 
612 	if (mlxsw_sp_acl_atcam_is_centry(aentry))
613 		err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
614 							      &aregion->cregion,
615 							      &aentry->centry,
616 							      rulei);
617 	else
618 		err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
619 								aregion, aentry,
620 								rulei);
621 
622 	return err;
623 }
624 
625 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
626 			    struct mlxsw_sp_acl_atcam *atcam)
627 {
628 	return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
629 }
630 
631 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
632 			     struct mlxsw_sp_acl_atcam *atcam)
633 {
634 	mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
635 }
636 
637 void *
638 mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
639 {
640 	return mlxsw_sp_acl_erp_rehash_hints_get(aregion);
641 }
642 
643 void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv)
644 {
645 	mlxsw_sp_acl_erp_rehash_hints_put(hints_priv);
646 }
647