1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
3 
4 #include <linux/kernel.h>
5 #include <linux/err.h>
6 #include <linux/errno.h>
7 #include <linux/gfp.h>
8 #include <linux/refcount.h>
9 #include <linux/rhashtable.h>
10 #define CREATE_TRACE_POINTS
11 #include <trace/events/mlxsw.h>
12 
13 #include "reg.h"
14 #include "core.h"
15 #include "spectrum.h"
16 #include "spectrum_acl_tcam.h"
17 #include "core_acl_flex_keys.h"
18 
19 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START	0
20 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END	5
21 
22 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
23 	char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
24 	u8 erp_id;
25 };
26 
27 struct mlxsw_sp_acl_atcam_lkey_id {
28 	struct rhash_head ht_node;
29 	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
30 	refcount_t refcnt;
31 	u32 id;
32 };
33 
34 struct mlxsw_sp_acl_atcam_region_ops {
35 	int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
36 	void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
37 	struct mlxsw_sp_acl_atcam_lkey_id *
38 		(*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
39 			       char *enc_key, u8 erp_id);
40 	void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
41 			    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
42 };
43 
44 struct mlxsw_sp_acl_atcam_region_generic {
45 	struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
46 };
47 
48 struct mlxsw_sp_acl_atcam_region_12kb {
49 	struct rhashtable lkey_ht;
50 	unsigned int max_lkey_id;
51 	unsigned long *used_lkey_id;
52 };
53 
54 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
55 	.key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
56 	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
57 	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
58 };
59 
60 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
61 	.key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
62 	.key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
63 	.head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
64 };
65 
66 static bool
mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry * aentry)67 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
68 {
69 	return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
70 }
71 
72 static int
mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region * aregion)73 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
74 {
75 	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
76 
77 	region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
78 	if (!region_generic)
79 		return -ENOMEM;
80 
81 	refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
82 	aregion->priv = region_generic;
83 
84 	return 0;
85 }
86 
87 static void
mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region * aregion)88 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
89 {
90 	kfree(aregion->priv);
91 }
92 
93 static struct mlxsw_sp_acl_atcam_lkey_id *
mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region * aregion,char * enc_key,u8 erp_id)94 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
95 				       char *enc_key, u8 erp_id)
96 {
97 	struct mlxsw_sp_acl_atcam_region_generic *region_generic;
98 
99 	region_generic = aregion->priv;
100 	return &region_generic->dummy_lkey_id;
101 }
102 
103 static void
mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_lkey_id * lkey_id)104 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
105 				       struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
106 {
107 }
108 
109 static const struct mlxsw_sp_acl_atcam_region_ops
110 mlxsw_sp_acl_atcam_region_generic_ops = {
111 	.init		= mlxsw_sp_acl_atcam_region_generic_init,
112 	.fini		= mlxsw_sp_acl_atcam_region_generic_fini,
113 	.lkey_id_get	= mlxsw_sp_acl_atcam_generic_lkey_id_get,
114 	.lkey_id_put	= mlxsw_sp_acl_atcam_generic_lkey_id_put,
115 };
116 
117 static int
mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region * aregion)118 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
119 {
120 	struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
121 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
122 	u64 max_lkey_id;
123 	int err;
124 
125 	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
126 		return -EIO;
127 
128 	max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
129 	region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
130 	if (!region_12kb)
131 		return -ENOMEM;
132 
133 	region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL);
134 	if (!region_12kb->used_lkey_id) {
135 		err = -ENOMEM;
136 		goto err_used_lkey_id_alloc;
137 	}
138 
139 	err = rhashtable_init(&region_12kb->lkey_ht,
140 			      &mlxsw_sp_acl_atcam_lkey_id_ht_params);
141 	if (err)
142 		goto err_rhashtable_init;
143 
144 	region_12kb->max_lkey_id = max_lkey_id;
145 	aregion->priv = region_12kb;
146 
147 	return 0;
148 
149 err_rhashtable_init:
150 	bitmap_free(region_12kb->used_lkey_id);
151 err_used_lkey_id_alloc:
152 	kfree(region_12kb);
153 	return err;
154 }
155 
156 static void
mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region * aregion)157 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
158 {
159 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
160 
161 	rhashtable_destroy(&region_12kb->lkey_ht);
162 	bitmap_free(region_12kb->used_lkey_id);
163 	kfree(region_12kb);
164 }
165 
166 static struct mlxsw_sp_acl_atcam_lkey_id *
mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_lkey_id_ht_key * ht_key)167 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
168 				  struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
169 {
170 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
171 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
172 	u32 id;
173 	int err;
174 
175 	id = find_first_zero_bit(region_12kb->used_lkey_id,
176 				 region_12kb->max_lkey_id);
177 	if (id < region_12kb->max_lkey_id)
178 		__set_bit(id, region_12kb->used_lkey_id);
179 	else
180 		return ERR_PTR(-ENOBUFS);
181 
182 	lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
183 	if (!lkey_id) {
184 		err = -ENOMEM;
185 		goto err_lkey_id_alloc;
186 	}
187 
188 	lkey_id->id = id;
189 	memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
190 	refcount_set(&lkey_id->refcnt, 1);
191 
192 	err = rhashtable_insert_fast(&region_12kb->lkey_ht,
193 				     &lkey_id->ht_node,
194 				     mlxsw_sp_acl_atcam_lkey_id_ht_params);
195 	if (err)
196 		goto err_rhashtable_insert;
197 
198 	return lkey_id;
199 
200 err_rhashtable_insert:
201 	kfree(lkey_id);
202 err_lkey_id_alloc:
203 	__clear_bit(id, region_12kb->used_lkey_id);
204 	return ERR_PTR(err);
205 }
206 
207 static void
mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_lkey_id * lkey_id)208 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
209 				   struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
210 {
211 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
212 	u32 id = lkey_id->id;
213 
214 	rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
215 			       mlxsw_sp_acl_atcam_lkey_id_ht_params);
216 	kfree(lkey_id);
217 	__clear_bit(id, region_12kb->used_lkey_id);
218 }
219 
220 static struct mlxsw_sp_acl_atcam_lkey_id *
mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region * aregion,char * enc_key,u8 erp_id)221 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
222 				    char *enc_key, u8 erp_id)
223 {
224 	struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
225 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
226 	struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
227 	struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
228 	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
229 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
230 
231 	memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
232 	mlxsw_afk_clear(afk, ht_key.enc_key,
233 			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
234 			MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
235 	ht_key.erp_id = erp_id;
236 	lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
237 					 mlxsw_sp_acl_atcam_lkey_id_ht_params);
238 	if (lkey_id) {
239 		refcount_inc(&lkey_id->refcnt);
240 		return lkey_id;
241 	}
242 
243 	return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
244 }
245 
246 static void
mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_lkey_id * lkey_id)247 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
248 				    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
249 {
250 	if (refcount_dec_and_test(&lkey_id->refcnt))
251 		mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
252 }
253 
254 static const struct mlxsw_sp_acl_atcam_region_ops
255 mlxsw_sp_acl_atcam_region_12kb_ops = {
256 	.init		= mlxsw_sp_acl_atcam_region_12kb_init,
257 	.fini		= mlxsw_sp_acl_atcam_region_12kb_fini,
258 	.lkey_id_get	= mlxsw_sp_acl_atcam_12kb_lkey_id_get,
259 	.lkey_id_put	= mlxsw_sp_acl_atcam_12kb_lkey_id_put,
260 };
261 
262 static const struct mlxsw_sp_acl_atcam_region_ops *
263 mlxsw_sp_acl_atcam_region_ops_arr[] = {
264 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]	=
265 		&mlxsw_sp_acl_atcam_region_generic_ops,
266 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]	=
267 		&mlxsw_sp_acl_atcam_region_generic_ops,
268 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]	=
269 		&mlxsw_sp_acl_atcam_region_generic_ops,
270 	[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]	=
271 		&mlxsw_sp_acl_atcam_region_12kb_ops,
272 };
273 
mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp * mlxsw_sp,u16 region_id)274 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
275 					u16 region_id)
276 {
277 	char perar_pl[MLXSW_REG_PERAR_LEN];
278 	/* For now, just assume that every region has 12 key blocks */
279 	u16 hw_region = region_id * 3;
280 	u64 max_regions;
281 
282 	max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
283 	if (hw_region >= max_regions)
284 		return -ENOBUFS;
285 
286 	mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
287 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
288 }
289 
290 static void
mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region * aregion)291 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
292 {
293 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
294 	enum mlxsw_sp_acl_atcam_region_type region_type;
295 	unsigned int blocks_count;
296 
297 	/* We already know the blocks count can not exceed the maximum
298 	 * blocks count.
299 	 */
300 	blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
301 	if (blocks_count <= 2)
302 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
303 	else if (blocks_count <= 4)
304 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
305 	else if (blocks_count <= 8)
306 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
307 	else
308 		region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
309 
310 	aregion->type = region_type;
311 	aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
312 }
313 
314 int
mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam * atcam,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_tcam_region * region,void * hints_priv,const struct mlxsw_sp_acl_ctcam_region_ops * ops)315 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
316 			       struct mlxsw_sp_acl_atcam *atcam,
317 			       struct mlxsw_sp_acl_atcam_region *aregion,
318 			       struct mlxsw_sp_acl_tcam_region *region,
319 			       void *hints_priv,
320 			       const struct mlxsw_sp_acl_ctcam_region_ops *ops)
321 {
322 	int err;
323 
324 	aregion->region = region;
325 	aregion->atcam = atcam;
326 	mlxsw_sp_acl_atcam_region_type_init(aregion);
327 	INIT_LIST_HEAD(&aregion->entries_list);
328 
329 	err = rhashtable_init(&aregion->entries_ht,
330 			      &mlxsw_sp_acl_atcam_entries_ht_params);
331 	if (err)
332 		return err;
333 	err = aregion->ops->init(aregion);
334 	if (err)
335 		goto err_ops_init;
336 	err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv);
337 	if (err)
338 		goto err_erp_region_init;
339 	err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
340 					     region, ops);
341 	if (err)
342 		goto err_ctcam_region_init;
343 
344 	return 0;
345 
346 err_ctcam_region_init:
347 	mlxsw_sp_acl_erp_region_fini(aregion);
348 err_erp_region_init:
349 	aregion->ops->fini(aregion);
350 err_ops_init:
351 	rhashtable_destroy(&aregion->entries_ht);
352 	return err;
353 }
354 
mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region * aregion)355 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
356 {
357 	mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
358 	mlxsw_sp_acl_erp_region_fini(aregion);
359 	aregion->ops->fini(aregion);
360 	rhashtable_destroy(&aregion->entries_ht);
361 	WARN_ON(!list_empty(&aregion->entries_list));
362 }
363 
mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_chunk * achunk,unsigned int priority)364 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
365 				   struct mlxsw_sp_acl_atcam_chunk *achunk,
366 				   unsigned int priority)
367 {
368 	mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
369 				      priority);
370 }
371 
mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk * achunk)372 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
373 {
374 	mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
375 }
376 
377 static int
mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)378 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
379 				       struct mlxsw_sp_acl_atcam_region *aregion,
380 				       struct mlxsw_sp_acl_atcam_entry *aentry,
381 				       struct mlxsw_sp_acl_rule_info *rulei)
382 {
383 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
384 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
385 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
386 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
387 	u32 kvdl_index, priority;
388 	int err;
389 
390 	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
391 	if (err)
392 		return err;
393 
394 	lkey_id = aregion->ops->lkey_id_get(aregion, aentry->ht_key.enc_key,
395 					    erp_id);
396 	if (IS_ERR(lkey_id))
397 		return PTR_ERR(lkey_id);
398 	aentry->lkey_id = lkey_id;
399 
400 	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
401 	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
402 			     priority, region->tcam_region_info,
403 			     aentry->ht_key.enc_key, erp_id,
404 			     aentry->delta_info.start,
405 			     aentry->delta_info.mask,
406 			     aentry->delta_info.value,
407 			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
408 			     kvdl_index);
409 	err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
410 	if (err)
411 		goto err_ptce3_write;
412 
413 	return 0;
414 
415 err_ptce3_write:
416 	aregion->ops->lkey_id_put(aregion, lkey_id);
417 	return err;
418 }
419 
420 static void
mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry)421 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
422 				       struct mlxsw_sp_acl_atcam_region *aregion,
423 				       struct mlxsw_sp_acl_atcam_entry *aentry)
424 {
425 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
426 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
427 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
428 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
429 
430 	mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
431 			     region->tcam_region_info,
432 			     aentry->ht_key.enc_key, erp_id,
433 			     aentry->delta_info.start,
434 			     aentry->delta_info.mask,
435 			     aentry->delta_info.value,
436 			     refcount_read(&lkey_id->refcnt) != 1,
437 			     lkey_id->id, 0);
438 	mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
439 	aregion->ops->lkey_id_put(aregion, lkey_id);
440 }
441 
442 static int
mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)443 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
444 					       struct mlxsw_sp_acl_atcam_region *aregion,
445 					       struct mlxsw_sp_acl_atcam_entry *aentry,
446 					       struct mlxsw_sp_acl_rule_info *rulei)
447 {
448 	struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
449 	u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
450 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
451 	char ptce3_pl[MLXSW_REG_PTCE3_LEN];
452 	u32 kvdl_index, priority;
453 	int err;
454 
455 	err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
456 	if (err)
457 		return err;
458 	kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
459 	mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
460 			     priority, region->tcam_region_info,
461 			     aentry->ht_key.enc_key, erp_id,
462 			     aentry->delta_info.start,
463 			     aentry->delta_info.mask,
464 			     aentry->delta_info.value,
465 			     refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
466 			     kvdl_index);
467 	return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
468 }
469 
470 static int
__mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)471 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
472 			       struct mlxsw_sp_acl_atcam_region *aregion,
473 			       struct mlxsw_sp_acl_atcam_entry *aentry,
474 			       struct mlxsw_sp_acl_rule_info *rulei)
475 {
476 	struct mlxsw_sp_acl_tcam_region *region = aregion->region;
477 	char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
478 	struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
479 	const struct mlxsw_sp_acl_erp_delta *delta;
480 	struct mlxsw_sp_acl_erp_mask *erp_mask;
481 	int err;
482 
483 	mlxsw_afk_encode(afk, region->key_info, &rulei->values,
484 			 aentry->ht_key.enc_key, mask);
485 
486 	erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
487 	if (IS_ERR(erp_mask))
488 		return PTR_ERR(erp_mask);
489 	aentry->erp_mask = erp_mask;
490 	aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
491 
492 	/* Compute all needed delta information and clear the delta bits
493 	 * from the encrypted key.
494 	 */
495 	delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
496 	aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
497 	aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
498 	aentry->delta_info.value =
499 		mlxsw_sp_acl_erp_delta_value(delta, aentry->ht_key.enc_key);
500 	mlxsw_sp_acl_erp_delta_clear(delta, aentry->ht_key.enc_key);
501 
502 	/* Add rule to the list of A-TCAM rules, assuming this
503 	 * rule is intended to A-TCAM. In case this rule does
504 	 * not fit into A-TCAM it will be removed from the list.
505 	 */
506 	list_add(&aentry->list, &aregion->entries_list);
507 
508 	/* We can't insert identical rules into the A-TCAM, so fail and
509 	 * let the rule spill into C-TCAM
510 	 */
511 	err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
512 					    &aentry->ht_node,
513 					    mlxsw_sp_acl_atcam_entries_ht_params);
514 	if (err)
515 		goto err_rhashtable_insert;
516 
517 	/* Bloom filter must be updated here, before inserting the rule into
518 	 * the A-TCAM.
519 	 */
520 	err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
521 	if (err)
522 		goto err_bf_insert;
523 
524 	err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
525 						     rulei);
526 	if (err)
527 		goto err_rule_insert;
528 
529 	return 0;
530 
531 err_rule_insert:
532 	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
533 err_bf_insert:
534 	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
535 			       mlxsw_sp_acl_atcam_entries_ht_params);
536 err_rhashtable_insert:
537 	list_del(&aentry->list);
538 	mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
539 	return err;
540 }
541 
542 static void
__mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry)543 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
544 			       struct mlxsw_sp_acl_atcam_region *aregion,
545 			       struct mlxsw_sp_acl_atcam_entry *aentry)
546 {
547 	mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
548 	mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
549 	rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
550 			       mlxsw_sp_acl_atcam_entries_ht_params);
551 	list_del(&aentry->list);
552 	mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
553 }
554 
555 static int
__mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)556 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
557 					  struct mlxsw_sp_acl_atcam_region *aregion,
558 					  struct mlxsw_sp_acl_atcam_entry *aentry,
559 					  struct mlxsw_sp_acl_rule_info *rulei)
560 {
561 	return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
562 							      aentry, rulei);
563 }
564 
mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_chunk * achunk,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)565 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
566 				 struct mlxsw_sp_acl_atcam_region *aregion,
567 				 struct mlxsw_sp_acl_atcam_chunk *achunk,
568 				 struct mlxsw_sp_acl_atcam_entry *aentry,
569 				 struct mlxsw_sp_acl_rule_info *rulei)
570 {
571 	int err;
572 
573 	err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
574 	if (!err)
575 		return 0;
576 
577 	/* It is possible we failed to add the rule to the A-TCAM due to
578 	 * exceeded number of masks. Try to spill into C-TCAM.
579 	 */
580 	trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion);
581 	err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
582 					   &achunk->cchunk, &aentry->centry,
583 					   rulei, true);
584 	if (!err)
585 		return 0;
586 
587 	return err;
588 }
589 
mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_chunk * achunk,struct mlxsw_sp_acl_atcam_entry * aentry)590 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
591 				  struct mlxsw_sp_acl_atcam_region *aregion,
592 				  struct mlxsw_sp_acl_atcam_chunk *achunk,
593 				  struct mlxsw_sp_acl_atcam_entry *aentry)
594 {
595 	if (mlxsw_sp_acl_atcam_is_centry(aentry))
596 		mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
597 					     &achunk->cchunk, &aentry->centry);
598 	else
599 		__mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
600 }
601 
602 int
mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam_region * aregion,struct mlxsw_sp_acl_atcam_entry * aentry,struct mlxsw_sp_acl_rule_info * rulei)603 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
604 					struct mlxsw_sp_acl_atcam_region *aregion,
605 					struct mlxsw_sp_acl_atcam_entry *aentry,
606 					struct mlxsw_sp_acl_rule_info *rulei)
607 {
608 	int err;
609 
610 	if (mlxsw_sp_acl_atcam_is_centry(aentry))
611 		err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
612 							      &aregion->cregion,
613 							      &aentry->centry,
614 							      rulei);
615 	else
616 		err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
617 								aregion, aentry,
618 								rulei);
619 
620 	return err;
621 }
622 
mlxsw_sp_acl_atcam_init(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam * atcam)623 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
624 			    struct mlxsw_sp_acl_atcam *atcam)
625 {
626 	return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
627 }
628 
mlxsw_sp_acl_atcam_fini(struct mlxsw_sp * mlxsw_sp,struct mlxsw_sp_acl_atcam * atcam)629 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
630 			     struct mlxsw_sp_acl_atcam *atcam)
631 {
632 	mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
633 }
634 
635 void *
mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region * aregion)636 mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
637 {
638 	return mlxsw_sp_acl_erp_rehash_hints_get(aregion);
639 }
640 
mlxsw_sp_acl_atcam_rehash_hints_put(void * hints_priv)641 void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv)
642 {
643 	mlxsw_sp_acl_erp_rehash_hints_put(hints_priv);
644 }
645