1 // SPDX-License-Identifier: GPL-2.0+
2 /* Microchip VCAP API
3  *
4  * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
5  */
6 
7 #include <linux/types.h>
8 
9 #include "vcap_api_private.h"
10 
11 static int keyfield_size_table[] = {
12 	[VCAP_FIELD_BIT]  = sizeof(struct vcap_u1_key),
13 	[VCAP_FIELD_U32]  = sizeof(struct vcap_u32_key),
14 	[VCAP_FIELD_U48]  = sizeof(struct vcap_u48_key),
15 	[VCAP_FIELD_U56]  = sizeof(struct vcap_u56_key),
16 	[VCAP_FIELD_U64]  = sizeof(struct vcap_u64_key),
17 	[VCAP_FIELD_U72]  = sizeof(struct vcap_u72_key),
18 	[VCAP_FIELD_U112] = sizeof(struct vcap_u112_key),
19 	[VCAP_FIELD_U128] = sizeof(struct vcap_u128_key),
20 };
21 
22 static int actionfield_size_table[] = {
23 	[VCAP_FIELD_BIT]  = sizeof(struct vcap_u1_action),
24 	[VCAP_FIELD_U32]  = sizeof(struct vcap_u32_action),
25 	[VCAP_FIELD_U48]  = sizeof(struct vcap_u48_action),
26 	[VCAP_FIELD_U56]  = sizeof(struct vcap_u56_action),
27 	[VCAP_FIELD_U64]  = sizeof(struct vcap_u64_action),
28 	[VCAP_FIELD_U72]  = sizeof(struct vcap_u72_action),
29 	[VCAP_FIELD_U112] = sizeof(struct vcap_u112_action),
30 	[VCAP_FIELD_U128] = sizeof(struct vcap_u128_action),
31 };
32 
33 /* Moving a rule in the VCAP address space */
34 struct vcap_rule_move {
35 	int addr; /* address to move */
36 	int offset; /* change in address */
37 	int count; /* blocksize of addresses to move */
38 };
39 
40 /* Stores the filter cookie and chain id that enabled the port */
41 struct vcap_enabled_port {
42 	struct list_head list; /* for insertion in enabled ports list */
43 	struct net_device *ndev;  /* the enabled port */
44 	unsigned long cookie; /* filter that enabled the port */
45 	int src_cid; /* source chain id */
46 	int dst_cid; /* destination chain id */
47 };
48 
49 void vcap_iter_set(struct vcap_stream_iter *itr, int sw_width,
50 		   const struct vcap_typegroup *tg, u32 offset)
51 {
52 	memset(itr, 0, sizeof(*itr));
53 	itr->offset = offset;
54 	itr->sw_width = sw_width;
55 	itr->regs_per_sw = DIV_ROUND_UP(sw_width, 32);
56 	itr->tg = tg;
57 }
58 
59 static void vcap_iter_skip_tg(struct vcap_stream_iter *itr)
60 {
61 	/* Compensate the field offset for preceding typegroups.
62 	 * A typegroup table ends with an all-zero terminator.
63 	 */
64 	while (itr->tg->width && itr->offset >= itr->tg->offset) {
65 		itr->offset += itr->tg->width;
66 		itr->tg++; /* next typegroup */
67 	}
68 }
69 
70 void vcap_iter_update(struct vcap_stream_iter *itr)
71 {
72 	int sw_idx, sw_bitpos;
73 
74 	/* Calculate the subword index and bitposition for current bit */
75 	sw_idx = itr->offset / itr->sw_width;
76 	sw_bitpos = itr->offset % itr->sw_width;
77 	/* Calculate the register index and bitposition for current bit */
78 	itr->reg_idx = (sw_idx * itr->regs_per_sw) + (sw_bitpos / 32);
79 	itr->reg_bitpos = sw_bitpos % 32;
80 }
81 
82 void vcap_iter_init(struct vcap_stream_iter *itr, int sw_width,
83 		    const struct vcap_typegroup *tg, u32 offset)
84 {
85 	vcap_iter_set(itr, sw_width, tg, offset);
86 	vcap_iter_skip_tg(itr);
87 	vcap_iter_update(itr);
88 }
89 
90 void vcap_iter_next(struct vcap_stream_iter *itr)
91 {
92 	itr->offset++;
93 	vcap_iter_skip_tg(itr);
94 	vcap_iter_update(itr);
95 }
96 
97 static void vcap_set_bit(u32 *stream, struct vcap_stream_iter *itr, bool value)
98 {
99 	u32 mask = BIT(itr->reg_bitpos);
100 	u32 *p = &stream[itr->reg_idx];
101 
102 	if (value)
103 		*p |= mask;
104 	else
105 		*p &= ~mask;
106 }
107 
108 static void vcap_encode_bit(u32 *stream, struct vcap_stream_iter *itr, bool val)
109 {
110 	/* When intersected by a type group field, stream the type group bits
111 	 * before continuing with the value bit
112 	 */
113 	while (itr->tg->width &&
114 	       itr->offset >= itr->tg->offset &&
115 	       itr->offset < itr->tg->offset + itr->tg->width) {
116 		int tg_bitpos = itr->tg->offset - itr->offset;
117 
118 		vcap_set_bit(stream, itr, (itr->tg->value >> tg_bitpos) & 0x1);
119 		itr->offset++;
120 		vcap_iter_update(itr);
121 	}
122 	vcap_set_bit(stream, itr, val);
123 }
124 
125 static void vcap_encode_field(u32 *stream, struct vcap_stream_iter *itr,
126 			      int width, const u8 *value)
127 {
128 	int idx;
129 
130 	/* Loop over the field value bits and add the value bits one by one to
131 	 * the output stream.
132 	 */
133 	for (idx = 0; idx < width; idx++) {
134 		u8 bidx = idx & GENMASK(2, 0);
135 
136 		/* Encode one field value bit */
137 		vcap_encode_bit(stream, itr, (value[idx / 8] >> bidx) & 0x1);
138 		vcap_iter_next(itr);
139 	}
140 }
141 
142 static void vcap_encode_typegroups(u32 *stream, int sw_width,
143 				   const struct vcap_typegroup *tg,
144 				   bool mask)
145 {
146 	struct vcap_stream_iter iter;
147 	int idx;
148 
149 	/* Mask bits must be set to zeros (inverted later when writing to the
150 	 * mask cache register), so that the mask typegroup bits consist of
151 	 * match-1 or match-0, or both
152 	 */
153 	vcap_iter_set(&iter, sw_width, tg, 0);
154 	while (iter.tg->width) {
155 		/* Set position to current typegroup bit */
156 		iter.offset = iter.tg->offset;
157 		vcap_iter_update(&iter);
158 		for (idx = 0; idx < iter.tg->width; idx++) {
159 			/* Iterate over current typegroup bits. Mask typegroup
160 			 * bits are always set
161 			 */
162 			if (mask)
163 				vcap_set_bit(stream, &iter, 0x1);
164 			else
165 				vcap_set_bit(stream, &iter,
166 					     (iter.tg->value >> idx) & 0x1);
167 			iter.offset++;
168 			vcap_iter_update(&iter);
169 		}
170 		iter.tg++; /* next typegroup */
171 	}
172 }
173 
174 static bool vcap_bitarray_zero(int width, u8 *value)
175 {
176 	int bytes = DIV_ROUND_UP(width, BITS_PER_BYTE);
177 	u8 total = 0, bmask = 0xff;
178 	int rwidth = width;
179 	int idx;
180 
181 	for (idx = 0; idx < bytes; ++idx, rwidth -= BITS_PER_BYTE) {
182 		if (rwidth && rwidth < BITS_PER_BYTE)
183 			bmask = (1 << rwidth) - 1;
184 		total += value[idx] & bmask;
185 	}
186 	return total == 0;
187 }
188 
189 static bool vcap_get_bit(u32 *stream, struct vcap_stream_iter *itr)
190 {
191 	u32 mask = BIT(itr->reg_bitpos);
192 	u32 *p = &stream[itr->reg_idx];
193 
194 	return !!(*p & mask);
195 }
196 
197 static void vcap_decode_field(u32 *stream, struct vcap_stream_iter *itr,
198 			      int width, u8 *value)
199 {
200 	int idx;
201 
202 	/* Loop over the field value bits and get the field bits and
203 	 * set them in the output value byte array
204 	 */
205 	for (idx = 0; idx < width; idx++) {
206 		u8 bidx = idx & 0x7;
207 
208 		/* Decode one field value bit */
209 		if (vcap_get_bit(stream, itr))
210 			*value |= 1 << bidx;
211 		vcap_iter_next(itr);
212 		if (bidx == 7)
213 			value++;
214 	}
215 }
216 
217 /* Verify that the type id in the stream matches the type id of the keyset */
218 static bool vcap_verify_keystream_keyset(struct vcap_control *vctrl,
219 					 enum vcap_type vt,
220 					 u32 *keystream,
221 					 u32 *mskstream,
222 					 enum vcap_keyfield_set keyset)
223 {
224 	const struct vcap_info *vcap = &vctrl->vcaps[vt];
225 	const struct vcap_field *typefld;
226 	const struct vcap_typegroup *tgt;
227 	const struct vcap_field *fields;
228 	struct vcap_stream_iter iter;
229 	const struct vcap_set *info;
230 	u32 value = 0;
231 	u32 mask = 0;
232 
233 	if (vcap_keyfield_count(vctrl, vt, keyset) == 0)
234 		return false;
235 
236 	info = vcap_keyfieldset(vctrl, vt, keyset);
237 	/* Check that the keyset is valid */
238 	if (!info)
239 		return false;
240 
241 	/* a type_id of value -1 means that there is no type field */
242 	if (info->type_id == (u8)-1)
243 		return true;
244 
245 	/* Get a valid typegroup for the specific keyset */
246 	tgt = vcap_keyfield_typegroup(vctrl, vt, keyset);
247 	if (!tgt)
248 		return false;
249 
250 	fields = vcap_keyfields(vctrl, vt, keyset);
251 	if (!fields)
252 		return false;
253 
254 	typefld = &fields[VCAP_KF_TYPE];
255 	vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset);
256 	vcap_decode_field(mskstream, &iter, typefld->width, (u8 *)&mask);
257 	/* no type info if there are no mask bits */
258 	if (vcap_bitarray_zero(typefld->width, (u8 *)&mask))
259 		return false;
260 
261 	/* Get the value of the type field in the stream and compare to the
262 	 * one define in the vcap keyset
263 	 */
264 	vcap_iter_init(&iter, vcap->sw_width, tgt, typefld->offset);
265 	vcap_decode_field(keystream, &iter, typefld->width, (u8 *)&value);
266 
267 	return (value & mask) == (info->type_id & mask);
268 }
269 
270 /* Verify that the typegroup bits have the correct values */
271 static int vcap_verify_typegroups(u32 *stream, int sw_width,
272 				  const struct vcap_typegroup *tgt, bool mask,
273 				  int sw_max)
274 {
275 	struct vcap_stream_iter iter;
276 	int sw_cnt, idx;
277 
278 	vcap_iter_set(&iter, sw_width, tgt, 0);
279 	sw_cnt = 0;
280 	while (iter.tg->width) {
281 		u32 value = 0;
282 		u32 tg_value = iter.tg->value;
283 
284 		if (mask)
285 			tg_value = (1 << iter.tg->width) - 1;
286 		/* Set position to current typegroup bit */
287 		iter.offset = iter.tg->offset;
288 		vcap_iter_update(&iter);
289 		for (idx = 0; idx < iter.tg->width; idx++) {
290 			/* Decode one typegroup bit */
291 			if (vcap_get_bit(stream, &iter))
292 				value |= 1 << idx;
293 			iter.offset++;
294 			vcap_iter_update(&iter);
295 		}
296 		if (value != tg_value)
297 			return -EINVAL;
298 		iter.tg++; /* next typegroup */
299 		sw_cnt++;
300 		/* Stop checking more typegroups */
301 		if (sw_max && sw_cnt >= sw_max)
302 			break;
303 	}
304 	return 0;
305 }
306 
307 /* Find the subword width of the key typegroup that matches the stream data */
308 static int vcap_find_keystream_typegroup_sw(struct vcap_control *vctrl,
309 					    enum vcap_type vt, u32 *stream,
310 					    bool mask, int sw_max)
311 {
312 	const struct vcap_typegroup **tgt;
313 	int sw_idx, res;
314 
315 	tgt = vctrl->vcaps[vt].keyfield_set_typegroups;
316 	/* Try the longest subword match first */
317 	for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) {
318 		if (!tgt[sw_idx])
319 			continue;
320 
321 		res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].sw_width,
322 					     tgt[sw_idx], mask, sw_max);
323 		if (res == 0)
324 			return sw_idx;
325 	}
326 	return -EINVAL;
327 }
328 
329 /* Verify that the typegroup information, subword count, keyset and type id
330  * are in sync and correct, return the list of matchin keysets
331  */
332 int
333 vcap_find_keystream_keysets(struct vcap_control *vctrl,
334 			    enum vcap_type vt,
335 			    u32 *keystream,
336 			    u32 *mskstream,
337 			    bool mask, int sw_max,
338 			    struct vcap_keyset_list *kslist)
339 {
340 	const struct vcap_set *keyfield_set;
341 	int sw_count, idx;
342 
343 	sw_count = vcap_find_keystream_typegroup_sw(vctrl, vt, keystream, mask,
344 						    sw_max);
345 	if (sw_count < 0)
346 		return sw_count;
347 
348 	keyfield_set = vctrl->vcaps[vt].keyfield_set;
349 	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
350 		if (keyfield_set[idx].sw_per_item != sw_count)
351 			continue;
352 
353 		if (vcap_verify_keystream_keyset(vctrl, vt, keystream,
354 						 mskstream, idx))
355 			vcap_keyset_list_add(kslist, idx);
356 	}
357 	if (kslist->cnt > 0)
358 		return 0;
359 	return -EINVAL;
360 }
361 EXPORT_SYMBOL_GPL(vcap_find_keystream_keysets);
362 
363 /* Read key data from a VCAP address and discover if there are any rule keysets
364  * here
365  */
366 int vcap_addr_keysets(struct vcap_control *vctrl,
367 		      struct net_device *ndev,
368 		      struct vcap_admin *admin,
369 		      int addr,
370 		      struct vcap_keyset_list *kslist)
371 {
372 	enum vcap_type vt = admin->vtype;
373 	int keyset_sw_regs, idx;
374 	u32 key = 0, mask = 0;
375 
376 	/* Read the cache at the specified address */
377 	keyset_sw_regs = DIV_ROUND_UP(vctrl->vcaps[vt].sw_width, 32);
378 	vctrl->ops->update(ndev, admin, VCAP_CMD_READ, VCAP_SEL_ALL, addr);
379 	vctrl->ops->cache_read(ndev, admin, VCAP_SEL_ENTRY, 0,
380 			       keyset_sw_regs);
381 	/* Skip uninitialized key/mask entries */
382 	for (idx = 0; idx < keyset_sw_regs; ++idx) {
383 		key |= ~admin->cache.keystream[idx];
384 		mask |= admin->cache.maskstream[idx];
385 	}
386 	if (key == 0 && mask == 0)
387 		return -EINVAL;
388 	/* Decode and locate the keysets */
389 	return vcap_find_keystream_keysets(vctrl, vt, admin->cache.keystream,
390 					   admin->cache.maskstream, false, 0,
391 					   kslist);
392 }
393 EXPORT_SYMBOL_GPL(vcap_addr_keysets);
394 
395 /* Return the list of keyfields for the keyset */
396 const struct vcap_field *vcap_keyfields(struct vcap_control *vctrl,
397 					enum vcap_type vt,
398 					enum vcap_keyfield_set keyset)
399 {
400 	/* Check that the keyset exists in the vcap keyset list */
401 	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
402 		return NULL;
403 	return vctrl->vcaps[vt].keyfield_set_map[keyset];
404 }
405 
406 /* Return the keyset information for the keyset */
407 const struct vcap_set *vcap_keyfieldset(struct vcap_control *vctrl,
408 					enum vcap_type vt,
409 					enum vcap_keyfield_set keyset)
410 {
411 	const struct vcap_set *kset;
412 
413 	/* Check that the keyset exists in the vcap keyset list */
414 	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
415 		return NULL;
416 	kset = &vctrl->vcaps[vt].keyfield_set[keyset];
417 	if (kset->sw_per_item == 0 || kset->sw_per_item > vctrl->vcaps[vt].sw_count)
418 		return NULL;
419 	return kset;
420 }
421 EXPORT_SYMBOL_GPL(vcap_keyfieldset);
422 
423 /* Return the typegroup table for the matching keyset (using subword size) */
424 const struct vcap_typegroup *
425 vcap_keyfield_typegroup(struct vcap_control *vctrl,
426 			enum vcap_type vt, enum vcap_keyfield_set keyset)
427 {
428 	const struct vcap_set *kset = vcap_keyfieldset(vctrl, vt, keyset);
429 
430 	/* Check that the keyset is valid */
431 	if (!kset)
432 		return NULL;
433 	return vctrl->vcaps[vt].keyfield_set_typegroups[kset->sw_per_item];
434 }
435 
436 /* Return the number of keyfields in the keyset */
437 int vcap_keyfield_count(struct vcap_control *vctrl,
438 			enum vcap_type vt, enum vcap_keyfield_set keyset)
439 {
440 	/* Check that the keyset exists in the vcap keyset list */
441 	if (keyset >= vctrl->vcaps[vt].keyfield_set_size)
442 		return 0;
443 	return vctrl->vcaps[vt].keyfield_set_map_size[keyset];
444 }
445 
446 static void vcap_encode_keyfield(struct vcap_rule_internal *ri,
447 				 const struct vcap_client_keyfield *kf,
448 				 const struct vcap_field *rf,
449 				 const struct vcap_typegroup *tgt)
450 {
451 	int sw_width = ri->vctrl->vcaps[ri->admin->vtype].sw_width;
452 	struct vcap_cache_data *cache = &ri->admin->cache;
453 	struct vcap_stream_iter iter;
454 	const u8 *value, *mask;
455 
456 	/* Encode the fields for the key and the mask in their respective
457 	 * streams, respecting the subword width.
458 	 */
459 	switch (kf->ctrl.type) {
460 	case VCAP_FIELD_BIT:
461 		value = &kf->data.u1.value;
462 		mask = &kf->data.u1.mask;
463 		break;
464 	case VCAP_FIELD_U32:
465 		value = (const u8 *)&kf->data.u32.value;
466 		mask = (const u8 *)&kf->data.u32.mask;
467 		break;
468 	case VCAP_FIELD_U48:
469 		value = kf->data.u48.value;
470 		mask = kf->data.u48.mask;
471 		break;
472 	case VCAP_FIELD_U56:
473 		value = kf->data.u56.value;
474 		mask = kf->data.u56.mask;
475 		break;
476 	case VCAP_FIELD_U64:
477 		value = kf->data.u64.value;
478 		mask = kf->data.u64.mask;
479 		break;
480 	case VCAP_FIELD_U72:
481 		value = kf->data.u72.value;
482 		mask = kf->data.u72.mask;
483 		break;
484 	case VCAP_FIELD_U112:
485 		value = kf->data.u112.value;
486 		mask = kf->data.u112.mask;
487 		break;
488 	case VCAP_FIELD_U128:
489 		value = kf->data.u128.value;
490 		mask = kf->data.u128.mask;
491 		break;
492 	}
493 	vcap_iter_init(&iter, sw_width, tgt, rf->offset);
494 	vcap_encode_field(cache->keystream, &iter, rf->width, value);
495 	vcap_iter_init(&iter, sw_width, tgt, rf->offset);
496 	vcap_encode_field(cache->maskstream, &iter, rf->width, mask);
497 }
498 
499 static void vcap_encode_keyfield_typegroups(struct vcap_control *vctrl,
500 					    struct vcap_rule_internal *ri,
501 					    const struct vcap_typegroup *tgt)
502 {
503 	int sw_width = vctrl->vcaps[ri->admin->vtype].sw_width;
504 	struct vcap_cache_data *cache = &ri->admin->cache;
505 
506 	/* Encode the typegroup bits for the key and the mask in their streams,
507 	 * respecting the subword width.
508 	 */
509 	vcap_encode_typegroups(cache->keystream, sw_width, tgt, false);
510 	vcap_encode_typegroups(cache->maskstream, sw_width, tgt, true);
511 }
512 
513 /* Copy data from src to dst but reverse the data in chunks of 32bits.
514  * For example if src is 00:11:22:33:44:55 where 55 is LSB the dst will
515  * have the value 22:33:44:55:00:11.
516  */
517 static void vcap_copy_to_w32be(u8 *dst, const u8 *src, int size)
518 {
519 	for (int idx = 0; idx < size; ++idx) {
520 		int first_byte_index = 0;
521 		int nidx;
522 
523 		first_byte_index = size - (((idx >> 2) + 1) << 2);
524 		if (first_byte_index < 0)
525 			first_byte_index = 0;
526 		nidx = idx + first_byte_index - (idx & ~0x3);
527 		dst[nidx] = src[idx];
528 	}
529 }
530 
531 static void
532 vcap_copy_from_client_keyfield(struct vcap_rule *rule,
533 			       struct vcap_client_keyfield *dst,
534 			       const struct vcap_client_keyfield *src)
535 {
536 	struct vcap_rule_internal *ri = to_intrule(rule);
537 	const struct vcap_client_keyfield_data *sdata;
538 	struct vcap_client_keyfield_data *ddata;
539 	int size;
540 
541 	dst->ctrl.type = src->ctrl.type;
542 	dst->ctrl.key = src->ctrl.key;
543 	INIT_LIST_HEAD(&dst->ctrl.list);
544 	sdata = &src->data;
545 	ddata = &dst->data;
546 
547 	if (!ri->admin->w32be) {
548 		memcpy(ddata, sdata, sizeof(dst->data));
549 		return;
550 	}
551 
552 	size = keyfield_size_table[dst->ctrl.type] / 2;
553 
554 	switch (dst->ctrl.type) {
555 	case VCAP_FIELD_BIT:
556 	case VCAP_FIELD_U32:
557 		memcpy(ddata, sdata, sizeof(dst->data));
558 		break;
559 	case VCAP_FIELD_U48:
560 		vcap_copy_to_w32be(ddata->u48.value, src->data.u48.value, size);
561 		vcap_copy_to_w32be(ddata->u48.mask,  src->data.u48.mask, size);
562 		break;
563 	case VCAP_FIELD_U56:
564 		vcap_copy_to_w32be(ddata->u56.value, sdata->u56.value, size);
565 		vcap_copy_to_w32be(ddata->u56.mask,  sdata->u56.mask, size);
566 		break;
567 	case VCAP_FIELD_U64:
568 		vcap_copy_to_w32be(ddata->u64.value, sdata->u64.value, size);
569 		vcap_copy_to_w32be(ddata->u64.mask,  sdata->u64.mask, size);
570 		break;
571 	case VCAP_FIELD_U72:
572 		vcap_copy_to_w32be(ddata->u72.value, sdata->u72.value, size);
573 		vcap_copy_to_w32be(ddata->u72.mask,  sdata->u72.mask, size);
574 		break;
575 	case VCAP_FIELD_U112:
576 		vcap_copy_to_w32be(ddata->u112.value, sdata->u112.value, size);
577 		vcap_copy_to_w32be(ddata->u112.mask,  sdata->u112.mask, size);
578 		break;
579 	case VCAP_FIELD_U128:
580 		vcap_copy_to_w32be(ddata->u128.value, sdata->u128.value, size);
581 		vcap_copy_to_w32be(ddata->u128.mask,  sdata->u128.mask, size);
582 		break;
583 	}
584 }
585 
586 static void
587 vcap_copy_from_client_actionfield(struct vcap_rule *rule,
588 				  struct vcap_client_actionfield *dst,
589 				  const struct vcap_client_actionfield *src)
590 {
591 	struct vcap_rule_internal *ri = to_intrule(rule);
592 	const struct vcap_client_actionfield_data *sdata;
593 	struct vcap_client_actionfield_data *ddata;
594 	int size;
595 
596 	dst->ctrl.type = src->ctrl.type;
597 	dst->ctrl.action = src->ctrl.action;
598 	INIT_LIST_HEAD(&dst->ctrl.list);
599 	sdata = &src->data;
600 	ddata = &dst->data;
601 
602 	if (!ri->admin->w32be) {
603 		memcpy(ddata, sdata, sizeof(dst->data));
604 		return;
605 	}
606 
607 	size = actionfield_size_table[dst->ctrl.type];
608 
609 	switch (dst->ctrl.type) {
610 	case VCAP_FIELD_BIT:
611 	case VCAP_FIELD_U32:
612 		memcpy(ddata, sdata, sizeof(dst->data));
613 		break;
614 	case VCAP_FIELD_U48:
615 		vcap_copy_to_w32be(ddata->u48.value, sdata->u48.value, size);
616 		break;
617 	case VCAP_FIELD_U56:
618 		vcap_copy_to_w32be(ddata->u56.value, sdata->u56.value, size);
619 		break;
620 	case VCAP_FIELD_U64:
621 		vcap_copy_to_w32be(ddata->u64.value, sdata->u64.value, size);
622 		break;
623 	case VCAP_FIELD_U72:
624 		vcap_copy_to_w32be(ddata->u72.value, sdata->u72.value, size);
625 		break;
626 	case VCAP_FIELD_U112:
627 		vcap_copy_to_w32be(ddata->u112.value, sdata->u112.value, size);
628 		break;
629 	case VCAP_FIELD_U128:
630 		vcap_copy_to_w32be(ddata->u128.value, sdata->u128.value, size);
631 		break;
632 	}
633 }
634 
635 static int vcap_encode_rule_keyset(struct vcap_rule_internal *ri)
636 {
637 	const struct vcap_client_keyfield *ckf;
638 	const struct vcap_typegroup *tg_table;
639 	struct vcap_client_keyfield tempkf;
640 	const struct vcap_field *kf_table;
641 	int keyset_size;
642 
643 	/* Get a valid set of fields for the specific keyset */
644 	kf_table = vcap_keyfields(ri->vctrl, ri->admin->vtype, ri->data.keyset);
645 	if (!kf_table) {
646 		pr_err("%s:%d: no fields available for this keyset: %d\n",
647 		       __func__, __LINE__, ri->data.keyset);
648 		return -EINVAL;
649 	}
650 	/* Get a valid typegroup for the specific keyset */
651 	tg_table = vcap_keyfield_typegroup(ri->vctrl, ri->admin->vtype,
652 					   ri->data.keyset);
653 	if (!tg_table) {
654 		pr_err("%s:%d: no typegroups available for this keyset: %d\n",
655 		       __func__, __LINE__, ri->data.keyset);
656 		return -EINVAL;
657 	}
658 	/* Get a valid size for the specific keyset */
659 	keyset_size = vcap_keyfield_count(ri->vctrl, ri->admin->vtype,
660 					  ri->data.keyset);
661 	if (keyset_size == 0) {
662 		pr_err("%s:%d: zero field count for this keyset: %d\n",
663 		       __func__, __LINE__, ri->data.keyset);
664 		return -EINVAL;
665 	}
666 	/* Iterate over the keyfields (key, mask) in the rule
667 	 * and encode these bits
668 	 */
669 	if (list_empty(&ri->data.keyfields)) {
670 		pr_err("%s:%d: no keyfields in the rule\n", __func__, __LINE__);
671 		return -EINVAL;
672 	}
673 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
674 		/* Check that the client entry exists in the keyset */
675 		if (ckf->ctrl.key >= keyset_size) {
676 			pr_err("%s:%d: key %d is not in vcap\n",
677 			       __func__, __LINE__, ckf->ctrl.key);
678 			return -EINVAL;
679 		}
680 		vcap_copy_from_client_keyfield(&ri->data, &tempkf, ckf);
681 		vcap_encode_keyfield(ri, &tempkf, &kf_table[ckf->ctrl.key],
682 				     tg_table);
683 	}
684 	/* Add typegroup bits to the key/mask bitstreams */
685 	vcap_encode_keyfield_typegroups(ri->vctrl, ri, tg_table);
686 	return 0;
687 }
688 
689 /* Return the list of actionfields for the actionset */
690 const struct vcap_field *
691 vcap_actionfields(struct vcap_control *vctrl,
692 		  enum vcap_type vt, enum vcap_actionfield_set actionset)
693 {
694 	/* Check that the actionset exists in the vcap actionset list */
695 	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
696 		return NULL;
697 	return vctrl->vcaps[vt].actionfield_set_map[actionset];
698 }
699 
700 const struct vcap_set *
701 vcap_actionfieldset(struct vcap_control *vctrl,
702 		    enum vcap_type vt, enum vcap_actionfield_set actionset)
703 {
704 	const struct vcap_set *aset;
705 
706 	/* Check that the actionset exists in the vcap actionset list */
707 	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
708 		return NULL;
709 	aset = &vctrl->vcaps[vt].actionfield_set[actionset];
710 	if (aset->sw_per_item == 0 || aset->sw_per_item > vctrl->vcaps[vt].sw_count)
711 		return NULL;
712 	return aset;
713 }
714 
715 /* Return the typegroup table for the matching actionset (using subword size) */
716 const struct vcap_typegroup *
717 vcap_actionfield_typegroup(struct vcap_control *vctrl,
718 			   enum vcap_type vt, enum vcap_actionfield_set actionset)
719 {
720 	const struct vcap_set *aset = vcap_actionfieldset(vctrl, vt, actionset);
721 
722 	/* Check that the actionset is valid */
723 	if (!aset)
724 		return NULL;
725 	return vctrl->vcaps[vt].actionfield_set_typegroups[aset->sw_per_item];
726 }
727 
728 /* Return the number of actionfields in the actionset */
729 int vcap_actionfield_count(struct vcap_control *vctrl,
730 			   enum vcap_type vt,
731 			   enum vcap_actionfield_set actionset)
732 {
733 	/* Check that the actionset exists in the vcap actionset list */
734 	if (actionset >= vctrl->vcaps[vt].actionfield_set_size)
735 		return 0;
736 	return vctrl->vcaps[vt].actionfield_set_map_size[actionset];
737 }
738 
739 static void vcap_encode_actionfield(struct vcap_rule_internal *ri,
740 				    const struct vcap_client_actionfield *af,
741 				    const struct vcap_field *rf,
742 				    const struct vcap_typegroup *tgt)
743 {
744 	int act_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
745 
746 	struct vcap_cache_data *cache = &ri->admin->cache;
747 	struct vcap_stream_iter iter;
748 	const u8 *value;
749 
750 	/* Encode the action field in the stream, respecting the subword width */
751 	switch (af->ctrl.type) {
752 	case VCAP_FIELD_BIT:
753 		value = &af->data.u1.value;
754 		break;
755 	case VCAP_FIELD_U32:
756 		value = (const u8 *)&af->data.u32.value;
757 		break;
758 	case VCAP_FIELD_U48:
759 		value = af->data.u48.value;
760 		break;
761 	case VCAP_FIELD_U56:
762 		value = af->data.u56.value;
763 		break;
764 	case VCAP_FIELD_U64:
765 		value = af->data.u64.value;
766 		break;
767 	case VCAP_FIELD_U72:
768 		value = af->data.u72.value;
769 		break;
770 	case VCAP_FIELD_U112:
771 		value = af->data.u112.value;
772 		break;
773 	case VCAP_FIELD_U128:
774 		value = af->data.u128.value;
775 		break;
776 	}
777 	vcap_iter_init(&iter, act_width, tgt, rf->offset);
778 	vcap_encode_field(cache->actionstream, &iter, rf->width, value);
779 }
780 
781 static void vcap_encode_actionfield_typegroups(struct vcap_rule_internal *ri,
782 					       const struct vcap_typegroup *tgt)
783 {
784 	int sw_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
785 	struct vcap_cache_data *cache = &ri->admin->cache;
786 
787 	/* Encode the typegroup bits for the actionstream respecting the subword
788 	 * width.
789 	 */
790 	vcap_encode_typegroups(cache->actionstream, sw_width, tgt, false);
791 }
792 
793 static int vcap_encode_rule_actionset(struct vcap_rule_internal *ri)
794 {
795 	const struct vcap_client_actionfield *caf;
796 	const struct vcap_typegroup *tg_table;
797 	struct vcap_client_actionfield tempaf;
798 	const struct vcap_field *af_table;
799 	int actionset_size;
800 
801 	/* Get a valid set of actionset fields for the specific actionset */
802 	af_table = vcap_actionfields(ri->vctrl, ri->admin->vtype,
803 				     ri->data.actionset);
804 	if (!af_table) {
805 		pr_err("%s:%d: no fields available for this actionset: %d\n",
806 		       __func__, __LINE__, ri->data.actionset);
807 		return -EINVAL;
808 	}
809 	/* Get a valid typegroup for the specific actionset */
810 	tg_table = vcap_actionfield_typegroup(ri->vctrl, ri->admin->vtype,
811 					      ri->data.actionset);
812 	if (!tg_table) {
813 		pr_err("%s:%d: no typegroups available for this actionset: %d\n",
814 		       __func__, __LINE__, ri->data.actionset);
815 		return -EINVAL;
816 	}
817 	/* Get a valid actionset size for the specific actionset */
818 	actionset_size = vcap_actionfield_count(ri->vctrl, ri->admin->vtype,
819 						ri->data.actionset);
820 	if (actionset_size == 0) {
821 		pr_err("%s:%d: zero field count for this actionset: %d\n",
822 		       __func__, __LINE__, ri->data.actionset);
823 		return -EINVAL;
824 	}
825 	/* Iterate over the actionfields in the rule
826 	 * and encode these bits
827 	 */
828 	if (list_empty(&ri->data.actionfields))
829 		pr_warn("%s:%d: no actionfields in the rule\n",
830 			__func__, __LINE__);
831 	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
832 		/* Check that the client action exists in the actionset */
833 		if (caf->ctrl.action >= actionset_size) {
834 			pr_err("%s:%d: action %d is not in vcap\n",
835 			       __func__, __LINE__, caf->ctrl.action);
836 			return -EINVAL;
837 		}
838 		vcap_copy_from_client_actionfield(&ri->data, &tempaf, caf);
839 		vcap_encode_actionfield(ri, &tempaf,
840 					&af_table[caf->ctrl.action], tg_table);
841 	}
842 	/* Add typegroup bits to the entry bitstreams */
843 	vcap_encode_actionfield_typegroups(ri, tg_table);
844 	return 0;
845 }
846 
847 static int vcap_encode_rule(struct vcap_rule_internal *ri)
848 {
849 	int err;
850 
851 	err = vcap_encode_rule_keyset(ri);
852 	if (err)
853 		return err;
854 	err = vcap_encode_rule_actionset(ri);
855 	if (err)
856 		return err;
857 	return 0;
858 }
859 
860 int vcap_api_check(struct vcap_control *ctrl)
861 {
862 	if (!ctrl) {
863 		pr_err("%s:%d: vcap control is missing\n", __func__, __LINE__);
864 		return -EINVAL;
865 	}
866 	if (!ctrl->ops || !ctrl->ops->validate_keyset ||
867 	    !ctrl->ops->add_default_fields || !ctrl->ops->cache_erase ||
868 	    !ctrl->ops->cache_write || !ctrl->ops->cache_read ||
869 	    !ctrl->ops->init || !ctrl->ops->update || !ctrl->ops->move ||
870 	    !ctrl->ops->port_info) {
871 		pr_err("%s:%d: client operations are missing\n",
872 		       __func__, __LINE__);
873 		return -ENOENT;
874 	}
875 	return 0;
876 }
877 
878 void vcap_erase_cache(struct vcap_rule_internal *ri)
879 {
880 	ri->vctrl->ops->cache_erase(ri->admin);
881 }
882 
883 /* Update the keyset for the rule */
884 int vcap_set_rule_set_keyset(struct vcap_rule *rule,
885 			     enum vcap_keyfield_set keyset)
886 {
887 	struct vcap_rule_internal *ri = to_intrule(rule);
888 	const struct vcap_set *kset;
889 	int sw_width;
890 
891 	kset = vcap_keyfieldset(ri->vctrl, ri->admin->vtype, keyset);
892 	/* Check that the keyset is valid */
893 	if (!kset)
894 		return -EINVAL;
895 	ri->keyset_sw = kset->sw_per_item;
896 	sw_width = ri->vctrl->vcaps[ri->admin->vtype].sw_width;
897 	ri->keyset_sw_regs = DIV_ROUND_UP(sw_width, 32);
898 	ri->data.keyset = keyset;
899 	return 0;
900 }
901 EXPORT_SYMBOL_GPL(vcap_set_rule_set_keyset);
902 
903 /* Update the actionset for the rule */
904 int vcap_set_rule_set_actionset(struct vcap_rule *rule,
905 				enum vcap_actionfield_set actionset)
906 {
907 	struct vcap_rule_internal *ri = to_intrule(rule);
908 	const struct vcap_set *aset;
909 	int act_width;
910 
911 	aset = vcap_actionfieldset(ri->vctrl, ri->admin->vtype, actionset);
912 	/* Check that the actionset is valid */
913 	if (!aset)
914 		return -EINVAL;
915 	ri->actionset_sw = aset->sw_per_item;
916 	act_width = ri->vctrl->vcaps[ri->admin->vtype].act_width;
917 	ri->actionset_sw_regs = DIV_ROUND_UP(act_width, 32);
918 	ri->data.actionset = actionset;
919 	return 0;
920 }
921 EXPORT_SYMBOL_GPL(vcap_set_rule_set_actionset);
922 
923 /* Check if a rule with this id exists */
924 static bool vcap_rule_exists(struct vcap_control *vctrl, u32 id)
925 {
926 	struct vcap_rule_internal *ri;
927 	struct vcap_admin *admin;
928 
929 	/* Look for the rule id in all vcaps */
930 	list_for_each_entry(admin, &vctrl->list, list)
931 		list_for_each_entry(ri, &admin->rules, list)
932 			if (ri->data.id == id)
933 				return true;
934 	return false;
935 }
936 
937 /* Find a rule with a provided rule id return a locked vcap */
938 static struct vcap_rule_internal *
939 vcap_get_locked_rule(struct vcap_control *vctrl, u32 id)
940 {
941 	struct vcap_rule_internal *ri;
942 	struct vcap_admin *admin;
943 
944 	/* Look for the rule id in all vcaps */
945 	list_for_each_entry(admin, &vctrl->list, list) {
946 		mutex_lock(&admin->lock);
947 		list_for_each_entry(ri, &admin->rules, list)
948 			if (ri->data.id == id)
949 				return ri;
950 		mutex_unlock(&admin->lock);
951 	}
952 	return NULL;
953 }
954 
955 /* Find a rule id with a provided cookie */
956 int vcap_lookup_rule_by_cookie(struct vcap_control *vctrl, u64 cookie)
957 {
958 	struct vcap_rule_internal *ri;
959 	struct vcap_admin *admin;
960 	int id = 0;
961 
962 	/* Look for the rule id in all vcaps */
963 	list_for_each_entry(admin, &vctrl->list, list) {
964 		mutex_lock(&admin->lock);
965 		list_for_each_entry(ri, &admin->rules, list) {
966 			if (ri->data.cookie == cookie) {
967 				id = ri->data.id;
968 				break;
969 			}
970 		}
971 		mutex_unlock(&admin->lock);
972 		if (id)
973 			return id;
974 	}
975 	return -ENOENT;
976 }
977 EXPORT_SYMBOL_GPL(vcap_lookup_rule_by_cookie);
978 
979 /* Get number of rules in a vcap instance lookup chain id range */
980 int vcap_admin_rule_count(struct vcap_admin *admin, int cid)
981 {
982 	int max_cid = roundup(cid + 1, VCAP_CID_LOOKUP_SIZE);
983 	int min_cid = rounddown(cid, VCAP_CID_LOOKUP_SIZE);
984 	struct vcap_rule_internal *elem;
985 	int count = 0;
986 
987 	list_for_each_entry(elem, &admin->rules, list) {
988 		mutex_lock(&admin->lock);
989 		if (elem->data.vcap_chain_id >= min_cid &&
990 		    elem->data.vcap_chain_id < max_cid)
991 			++count;
992 		mutex_unlock(&admin->lock);
993 	}
994 	return count;
995 }
996 EXPORT_SYMBOL_GPL(vcap_admin_rule_count);
997 
998 /* Make a copy of the rule, shallow or full */
999 static struct vcap_rule_internal *vcap_dup_rule(struct vcap_rule_internal *ri,
1000 						bool full)
1001 {
1002 	struct vcap_client_actionfield *caf, *newcaf;
1003 	struct vcap_client_keyfield *ckf, *newckf;
1004 	struct vcap_rule_internal *duprule;
1005 
1006 	/* Allocate the client part */
1007 	duprule = kzalloc(sizeof(*duprule), GFP_KERNEL);
1008 	if (!duprule)
1009 		return ERR_PTR(-ENOMEM);
1010 	*duprule = *ri;
1011 	/* Not inserted in the VCAP */
1012 	INIT_LIST_HEAD(&duprule->list);
1013 	/* No elements in these lists */
1014 	INIT_LIST_HEAD(&duprule->data.keyfields);
1015 	INIT_LIST_HEAD(&duprule->data.actionfields);
1016 
1017 	/* A full rule copy includes keys and actions */
1018 	if (!full)
1019 		return duprule;
1020 
1021 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
1022 		newckf = kmemdup(ckf, sizeof(*newckf), GFP_KERNEL);
1023 		if (!newckf)
1024 			return ERR_PTR(-ENOMEM);
1025 		list_add_tail(&newckf->ctrl.list, &duprule->data.keyfields);
1026 	}
1027 
1028 	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
1029 		newcaf = kmemdup(caf, sizeof(*newcaf), GFP_KERNEL);
1030 		if (!newcaf)
1031 			return ERR_PTR(-ENOMEM);
1032 		list_add_tail(&newcaf->ctrl.list, &duprule->data.actionfields);
1033 	}
1034 
1035 	return duprule;
1036 }
1037 
1038 static void vcap_apply_width(u8 *dst, int width, int bytes)
1039 {
1040 	u8 bmask;
1041 	int idx;
1042 
1043 	for (idx = 0; idx < bytes; idx++) {
1044 		if (width > 0)
1045 			if (width < 8)
1046 				bmask = (1 << width) - 1;
1047 			else
1048 				bmask = ~0;
1049 		else
1050 			bmask = 0;
1051 		dst[idx] &= bmask;
1052 		width -= 8;
1053 	}
1054 }
1055 
1056 static void vcap_copy_from_w32be(u8 *dst, u8 *src, int size, int width)
1057 {
1058 	int idx, ridx, wstart, nidx;
1059 	int tail_bytes = (((size + 4) >> 2) << 2) - size;
1060 
1061 	for (idx = 0, ridx = size - 1; idx < size; ++idx, --ridx) {
1062 		wstart = (idx >> 2) << 2;
1063 		nidx = wstart + 3 - (idx & 0x3);
1064 		if (nidx >= size)
1065 			nidx -= tail_bytes;
1066 		dst[nidx] = src[ridx];
1067 	}
1068 
1069 	vcap_apply_width(dst, width, size);
1070 }
1071 
1072 static void vcap_copy_action_bit_field(struct vcap_u1_action *field, u8 *value)
1073 {
1074 	field->value = (*value) & 0x1;
1075 }
1076 
1077 static void vcap_copy_limited_actionfield(u8 *dstvalue, u8 *srcvalue,
1078 					  int width, int bytes)
1079 {
1080 	memcpy(dstvalue, srcvalue, bytes);
1081 	vcap_apply_width(dstvalue, width, bytes);
1082 }
1083 
1084 static void vcap_copy_to_client_actionfield(struct vcap_rule_internal *ri,
1085 					    struct vcap_client_actionfield *field,
1086 					    u8 *value, u16 width)
1087 {
1088 	int field_size = actionfield_size_table[field->ctrl.type];
1089 
1090 	if (ri->admin->w32be) {
1091 		switch (field->ctrl.type) {
1092 		case VCAP_FIELD_BIT:
1093 			vcap_copy_action_bit_field(&field->data.u1, value);
1094 			break;
1095 		case VCAP_FIELD_U32:
1096 			vcap_copy_limited_actionfield((u8 *)&field->data.u32.value,
1097 						      value,
1098 						      width, field_size);
1099 			break;
1100 		case VCAP_FIELD_U48:
1101 			vcap_copy_from_w32be(field->data.u48.value, value,
1102 					     field_size, width);
1103 			break;
1104 		case VCAP_FIELD_U56:
1105 			vcap_copy_from_w32be(field->data.u56.value, value,
1106 					     field_size, width);
1107 			break;
1108 		case VCAP_FIELD_U64:
1109 			vcap_copy_from_w32be(field->data.u64.value, value,
1110 					     field_size, width);
1111 			break;
1112 		case VCAP_FIELD_U72:
1113 			vcap_copy_from_w32be(field->data.u72.value, value,
1114 					     field_size, width);
1115 			break;
1116 		case VCAP_FIELD_U112:
1117 			vcap_copy_from_w32be(field->data.u112.value, value,
1118 					     field_size, width);
1119 			break;
1120 		case VCAP_FIELD_U128:
1121 			vcap_copy_from_w32be(field->data.u128.value, value,
1122 					     field_size, width);
1123 			break;
1124 		}
1125 	} else {
1126 		switch (field->ctrl.type) {
1127 		case VCAP_FIELD_BIT:
1128 			vcap_copy_action_bit_field(&field->data.u1, value);
1129 			break;
1130 		case VCAP_FIELD_U32:
1131 			vcap_copy_limited_actionfield((u8 *)&field->data.u32.value,
1132 						      value,
1133 						      width, field_size);
1134 			break;
1135 		case VCAP_FIELD_U48:
1136 			vcap_copy_limited_actionfield(field->data.u48.value,
1137 						      value,
1138 						      width, field_size);
1139 			break;
1140 		case VCAP_FIELD_U56:
1141 			vcap_copy_limited_actionfield(field->data.u56.value,
1142 						      value,
1143 						      width, field_size);
1144 			break;
1145 		case VCAP_FIELD_U64:
1146 			vcap_copy_limited_actionfield(field->data.u64.value,
1147 						      value,
1148 						      width, field_size);
1149 			break;
1150 		case VCAP_FIELD_U72:
1151 			vcap_copy_limited_actionfield(field->data.u72.value,
1152 						      value,
1153 						      width, field_size);
1154 			break;
1155 		case VCAP_FIELD_U112:
1156 			vcap_copy_limited_actionfield(field->data.u112.value,
1157 						      value,
1158 						      width, field_size);
1159 			break;
1160 		case VCAP_FIELD_U128:
1161 			vcap_copy_limited_actionfield(field->data.u128.value,
1162 						      value,
1163 						      width, field_size);
1164 			break;
1165 		}
1166 	}
1167 }
1168 
1169 static void vcap_copy_key_bit_field(struct vcap_u1_key *field,
1170 				    u8 *value, u8 *mask)
1171 {
1172 	field->value = (*value) & 0x1;
1173 	field->mask = (*mask) & 0x1;
1174 }
1175 
1176 static void vcap_copy_limited_keyfield(u8 *dstvalue, u8 *dstmask,
1177 				       u8 *srcvalue, u8 *srcmask,
1178 				       int width, int bytes)
1179 {
1180 	memcpy(dstvalue, srcvalue, bytes);
1181 	vcap_apply_width(dstvalue, width, bytes);
1182 	memcpy(dstmask, srcmask, bytes);
1183 	vcap_apply_width(dstmask, width, bytes);
1184 }
1185 
1186 static void vcap_copy_to_client_keyfield(struct vcap_rule_internal *ri,
1187 					 struct vcap_client_keyfield *field,
1188 					 u8 *value, u8 *mask, u16 width)
1189 {
1190 	int field_size = keyfield_size_table[field->ctrl.type] / 2;
1191 
1192 	if (ri->admin->w32be) {
1193 		switch (field->ctrl.type) {
1194 		case VCAP_FIELD_BIT:
1195 			vcap_copy_key_bit_field(&field->data.u1, value, mask);
1196 			break;
1197 		case VCAP_FIELD_U32:
1198 			vcap_copy_limited_keyfield((u8 *)&field->data.u32.value,
1199 						   (u8 *)&field->data.u32.mask,
1200 						   value, mask,
1201 						   width, field_size);
1202 			break;
1203 		case VCAP_FIELD_U48:
1204 			vcap_copy_from_w32be(field->data.u48.value, value,
1205 					     field_size, width);
1206 			vcap_copy_from_w32be(field->data.u48.mask,  mask,
1207 					     field_size, width);
1208 			break;
1209 		case VCAP_FIELD_U56:
1210 			vcap_copy_from_w32be(field->data.u56.value, value,
1211 					     field_size, width);
1212 			vcap_copy_from_w32be(field->data.u56.mask,  mask,
1213 					     field_size, width);
1214 			break;
1215 		case VCAP_FIELD_U64:
1216 			vcap_copy_from_w32be(field->data.u64.value, value,
1217 					     field_size, width);
1218 			vcap_copy_from_w32be(field->data.u64.mask,  mask,
1219 					     field_size, width);
1220 			break;
1221 		case VCAP_FIELD_U72:
1222 			vcap_copy_from_w32be(field->data.u72.value, value,
1223 					     field_size, width);
1224 			vcap_copy_from_w32be(field->data.u72.mask,  mask,
1225 					     field_size, width);
1226 			break;
1227 		case VCAP_FIELD_U112:
1228 			vcap_copy_from_w32be(field->data.u112.value, value,
1229 					     field_size, width);
1230 			vcap_copy_from_w32be(field->data.u112.mask,  mask,
1231 					     field_size, width);
1232 			break;
1233 		case VCAP_FIELD_U128:
1234 			vcap_copy_from_w32be(field->data.u128.value, value,
1235 					     field_size, width);
1236 			vcap_copy_from_w32be(field->data.u128.mask,  mask,
1237 					     field_size, width);
1238 			break;
1239 		}
1240 	} else {
1241 		switch (field->ctrl.type) {
1242 		case VCAP_FIELD_BIT:
1243 			vcap_copy_key_bit_field(&field->data.u1, value, mask);
1244 			break;
1245 		case VCAP_FIELD_U32:
1246 			vcap_copy_limited_keyfield((u8 *)&field->data.u32.value,
1247 						   (u8 *)&field->data.u32.mask,
1248 						   value, mask,
1249 						   width, field_size);
1250 			break;
1251 		case VCAP_FIELD_U48:
1252 			vcap_copy_limited_keyfield(field->data.u48.value,
1253 						   field->data.u48.mask,
1254 						   value, mask,
1255 						   width, field_size);
1256 			break;
1257 		case VCAP_FIELD_U56:
1258 			vcap_copy_limited_keyfield(field->data.u56.value,
1259 						   field->data.u56.mask,
1260 						   value, mask,
1261 						   width, field_size);
1262 			break;
1263 		case VCAP_FIELD_U64:
1264 			vcap_copy_limited_keyfield(field->data.u64.value,
1265 						   field->data.u64.mask,
1266 						   value, mask,
1267 						   width, field_size);
1268 			break;
1269 		case VCAP_FIELD_U72:
1270 			vcap_copy_limited_keyfield(field->data.u72.value,
1271 						   field->data.u72.mask,
1272 						   value, mask,
1273 						   width, field_size);
1274 			break;
1275 		case VCAP_FIELD_U112:
1276 			vcap_copy_limited_keyfield(field->data.u112.value,
1277 						   field->data.u112.mask,
1278 						   value, mask,
1279 						   width, field_size);
1280 			break;
1281 		case VCAP_FIELD_U128:
1282 			vcap_copy_limited_keyfield(field->data.u128.value,
1283 						   field->data.u128.mask,
1284 						   value, mask,
1285 						   width, field_size);
1286 			break;
1287 		}
1288 	}
1289 }
1290 
1291 static void vcap_rule_alloc_keyfield(struct vcap_rule_internal *ri,
1292 				     const struct vcap_field *keyfield,
1293 				     enum vcap_key_field key,
1294 				     u8 *value, u8 *mask)
1295 {
1296 	struct vcap_client_keyfield *field;
1297 
1298 	field = kzalloc(sizeof(*field), GFP_KERNEL);
1299 	if (!field)
1300 		return;
1301 	INIT_LIST_HEAD(&field->ctrl.list);
1302 	field->ctrl.key = key;
1303 	field->ctrl.type = keyfield->type;
1304 	vcap_copy_to_client_keyfield(ri, field, value, mask, keyfield->width);
1305 	list_add_tail(&field->ctrl.list, &ri->data.keyfields);
1306 }
1307 
1308 /* Read key data from a VCAP address and discover if there is a rule keyset
1309  * here
1310  */
1311 static bool
1312 vcap_verify_actionstream_actionset(struct vcap_control *vctrl,
1313 				   enum vcap_type vt,
1314 				   u32 *actionstream,
1315 				   enum vcap_actionfield_set actionset)
1316 {
1317 	const struct vcap_typegroup *tgt;
1318 	const struct vcap_field *fields;
1319 	const struct vcap_set *info;
1320 
1321 	if (vcap_actionfield_count(vctrl, vt, actionset) == 0)
1322 		return false;
1323 
1324 	info = vcap_actionfieldset(vctrl, vt, actionset);
1325 	/* Check that the actionset is valid */
1326 	if (!info)
1327 		return false;
1328 
1329 	/* a type_id of value -1 means that there is no type field */
1330 	if (info->type_id == (u8)-1)
1331 		return true;
1332 
1333 	/* Get a valid typegroup for the specific actionset */
1334 	tgt = vcap_actionfield_typegroup(vctrl, vt, actionset);
1335 	if (!tgt)
1336 		return false;
1337 
1338 	fields = vcap_actionfields(vctrl, vt, actionset);
1339 	if (!fields)
1340 		return false;
1341 
1342 	/* Later this will be expanded with a check of the type id */
1343 	return true;
1344 }
1345 
1346 /* Find the subword width of the action typegroup that matches the stream data
1347  */
1348 static int vcap_find_actionstream_typegroup_sw(struct vcap_control *vctrl,
1349 					       enum vcap_type vt, u32 *stream,
1350 					       int sw_max)
1351 {
1352 	const struct vcap_typegroup **tgt;
1353 	int sw_idx, res;
1354 
1355 	tgt = vctrl->vcaps[vt].actionfield_set_typegroups;
1356 	/* Try the longest subword match first */
1357 	for (sw_idx = vctrl->vcaps[vt].sw_count; sw_idx >= 0; sw_idx--) {
1358 		if (!tgt[sw_idx])
1359 			continue;
1360 		res = vcap_verify_typegroups(stream, vctrl->vcaps[vt].act_width,
1361 					     tgt[sw_idx], false, sw_max);
1362 		if (res == 0)
1363 			return sw_idx;
1364 	}
1365 	return -EINVAL;
1366 }
1367 
1368 /* Verify that the typegroup information, subword count, actionset and type id
1369  * are in sync and correct, return the actionset
1370  */
1371 static enum vcap_actionfield_set
1372 vcap_find_actionstream_actionset(struct vcap_control *vctrl,
1373 				 enum vcap_type vt,
1374 				 u32 *stream,
1375 				 int sw_max)
1376 {
1377 	const struct vcap_set *actionfield_set;
1378 	int sw_count, idx;
1379 	bool res;
1380 
1381 	sw_count = vcap_find_actionstream_typegroup_sw(vctrl, vt, stream,
1382 						       sw_max);
1383 	if (sw_count < 0)
1384 		return sw_count;
1385 
1386 	actionfield_set = vctrl->vcaps[vt].actionfield_set;
1387 	for (idx = 0; idx < vctrl->vcaps[vt].actionfield_set_size; ++idx) {
1388 		if (actionfield_set[idx].sw_per_item != sw_count)
1389 			continue;
1390 
1391 		res = vcap_verify_actionstream_actionset(vctrl, vt,
1392 							 stream, idx);
1393 		if (res)
1394 			return idx;
1395 	}
1396 	return -EINVAL;
1397 }
1398 
1399 /* Store action value in an element in a list for the client */
1400 static void vcap_rule_alloc_actionfield(struct vcap_rule_internal *ri,
1401 					const struct vcap_field *actionfield,
1402 					enum vcap_action_field action,
1403 					u8 *value)
1404 {
1405 	struct vcap_client_actionfield *field;
1406 
1407 	field = kzalloc(sizeof(*field), GFP_KERNEL);
1408 	if (!field)
1409 		return;
1410 	INIT_LIST_HEAD(&field->ctrl.list);
1411 	field->ctrl.action = action;
1412 	field->ctrl.type = actionfield->type;
1413 	vcap_copy_to_client_actionfield(ri, field, value, actionfield->width);
1414 	list_add_tail(&field->ctrl.list, &ri->data.actionfields);
1415 }
1416 
1417 static int vcap_decode_actionset(struct vcap_rule_internal *ri)
1418 {
1419 	struct vcap_control *vctrl = ri->vctrl;
1420 	struct vcap_admin *admin = ri->admin;
1421 	const struct vcap_field *actionfield;
1422 	enum vcap_actionfield_set actionset;
1423 	enum vcap_type vt = admin->vtype;
1424 	const struct vcap_typegroup *tgt;
1425 	struct vcap_stream_iter iter;
1426 	int idx, res, actfield_count;
1427 	u32 *actstream;
1428 	u8 value[16];
1429 
1430 	actstream = admin->cache.actionstream;
1431 	res = vcap_find_actionstream_actionset(vctrl, vt, actstream, 0);
1432 	if (res < 0) {
1433 		pr_err("%s:%d: could not find valid actionset: %d\n",
1434 		       __func__, __LINE__, res);
1435 		return -EINVAL;
1436 	}
1437 	actionset = res;
1438 	actfield_count = vcap_actionfield_count(vctrl, vt, actionset);
1439 	actionfield = vcap_actionfields(vctrl, vt, actionset);
1440 	tgt = vcap_actionfield_typegroup(vctrl, vt, actionset);
1441 	/* Start decoding the stream */
1442 	for (idx = 0; idx < actfield_count; ++idx) {
1443 		if (actionfield[idx].width <= 0)
1444 			continue;
1445 		/* Get the action */
1446 		memset(value, 0, DIV_ROUND_UP(actionfield[idx].width, 8));
1447 		vcap_iter_init(&iter, vctrl->vcaps[vt].act_width, tgt,
1448 			       actionfield[idx].offset);
1449 		vcap_decode_field(actstream, &iter, actionfield[idx].width,
1450 				  value);
1451 		/* Skip if no bits are set */
1452 		if (vcap_bitarray_zero(actionfield[idx].width, value))
1453 			continue;
1454 		vcap_rule_alloc_actionfield(ri, &actionfield[idx], idx, value);
1455 		/* Later the action id will also be checked */
1456 	}
1457 	return vcap_set_rule_set_actionset((struct vcap_rule *)ri, actionset);
1458 }
1459 
1460 static int vcap_decode_keyset(struct vcap_rule_internal *ri)
1461 {
1462 	struct vcap_control *vctrl = ri->vctrl;
1463 	struct vcap_stream_iter kiter, miter;
1464 	struct vcap_admin *admin = ri->admin;
1465 	enum vcap_keyfield_set keysets[10];
1466 	const struct vcap_field *keyfield;
1467 	enum vcap_type vt = admin->vtype;
1468 	const struct vcap_typegroup *tgt;
1469 	struct vcap_keyset_list matches;
1470 	enum vcap_keyfield_set keyset;
1471 	int idx, res, keyfield_count;
1472 	u32 *maskstream;
1473 	u32 *keystream;
1474 	u8 value[16];
1475 	u8 mask[16];
1476 
1477 	keystream = admin->cache.keystream;
1478 	maskstream = admin->cache.maskstream;
1479 	matches.keysets = keysets;
1480 	matches.cnt = 0;
1481 	matches.max = ARRAY_SIZE(keysets);
1482 	res = vcap_find_keystream_keysets(vctrl, vt, keystream, maskstream,
1483 					  false, 0, &matches);
1484 	if (res < 0) {
1485 		pr_err("%s:%d: could not find valid keysets: %d\n",
1486 		       __func__, __LINE__, res);
1487 		return -EINVAL;
1488 	}
1489 	keyset = matches.keysets[0];
1490 	keyfield_count = vcap_keyfield_count(vctrl, vt, keyset);
1491 	keyfield = vcap_keyfields(vctrl, vt, keyset);
1492 	tgt = vcap_keyfield_typegroup(vctrl, vt, keyset);
1493 	/* Start decoding the streams */
1494 	for (idx = 0; idx < keyfield_count; ++idx) {
1495 		if (keyfield[idx].width <= 0)
1496 			continue;
1497 		/* First get the mask */
1498 		memset(mask, 0, DIV_ROUND_UP(keyfield[idx].width, 8));
1499 		vcap_iter_init(&miter, vctrl->vcaps[vt].sw_width, tgt,
1500 			       keyfield[idx].offset);
1501 		vcap_decode_field(maskstream, &miter, keyfield[idx].width,
1502 				  mask);
1503 		/* Skip if no mask bits are set */
1504 		if (vcap_bitarray_zero(keyfield[idx].width, mask))
1505 			continue;
1506 		/* Get the key */
1507 		memset(value, 0, DIV_ROUND_UP(keyfield[idx].width, 8));
1508 		vcap_iter_init(&kiter, vctrl->vcaps[vt].sw_width, tgt,
1509 			       keyfield[idx].offset);
1510 		vcap_decode_field(keystream, &kiter, keyfield[idx].width,
1511 				  value);
1512 		vcap_rule_alloc_keyfield(ri, &keyfield[idx], idx, value, mask);
1513 	}
1514 	return vcap_set_rule_set_keyset((struct vcap_rule *)ri, keyset);
1515 }
1516 
1517 /* Read VCAP content into the VCAP cache */
1518 static int vcap_read_rule(struct vcap_rule_internal *ri)
1519 {
1520 	struct vcap_admin *admin = ri->admin;
1521 	int sw_idx, ent_idx = 0, act_idx = 0;
1522 	u32 addr = ri->addr;
1523 
1524 	if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) {
1525 		pr_err("%s:%d: rule is empty\n", __func__, __LINE__);
1526 		return -EINVAL;
1527 	}
1528 	vcap_erase_cache(ri);
1529 	/* Use the values in the streams to read the VCAP cache */
1530 	for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) {
1531 		ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ,
1532 				       VCAP_SEL_ALL, addr);
1533 		ri->vctrl->ops->cache_read(ri->ndev, admin,
1534 					   VCAP_SEL_ENTRY, ent_idx,
1535 					   ri->keyset_sw_regs);
1536 		ri->vctrl->ops->cache_read(ri->ndev, admin,
1537 					   VCAP_SEL_ACTION, act_idx,
1538 					   ri->actionset_sw_regs);
1539 		if (sw_idx == 0)
1540 			ri->vctrl->ops->cache_read(ri->ndev, admin,
1541 						   VCAP_SEL_COUNTER,
1542 						   ri->counter_id, 0);
1543 		ent_idx += ri->keyset_sw_regs;
1544 		act_idx += ri->actionset_sw_regs;
1545 	}
1546 	return 0;
1547 }
1548 
1549 /* Write VCAP cache content to the VCAP HW instance */
1550 static int vcap_write_rule(struct vcap_rule_internal *ri)
1551 {
1552 	struct vcap_admin *admin = ri->admin;
1553 	int sw_idx, ent_idx = 0, act_idx = 0;
1554 	u32 addr = ri->addr;
1555 
1556 	if (!ri->size || !ri->keyset_sw_regs || !ri->actionset_sw_regs) {
1557 		pr_err("%s:%d: rule is empty\n", __func__, __LINE__);
1558 		return -EINVAL;
1559 	}
1560 	/* Use the values in the streams to write the VCAP cache */
1561 	for (sw_idx = 0; sw_idx < ri->size; sw_idx++, addr++) {
1562 		ri->vctrl->ops->cache_write(ri->ndev, admin,
1563 					    VCAP_SEL_ENTRY, ent_idx,
1564 					    ri->keyset_sw_regs);
1565 		ri->vctrl->ops->cache_write(ri->ndev, admin,
1566 					    VCAP_SEL_ACTION, act_idx,
1567 					    ri->actionset_sw_regs);
1568 		ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_WRITE,
1569 				       VCAP_SEL_ALL, addr);
1570 		ent_idx += ri->keyset_sw_regs;
1571 		act_idx += ri->actionset_sw_regs;
1572 	}
1573 	return 0;
1574 }
1575 
1576 static int vcap_write_counter(struct vcap_rule_internal *ri,
1577 			      struct vcap_counter *ctr)
1578 {
1579 	struct vcap_admin *admin = ri->admin;
1580 
1581 	admin->cache.counter = ctr->value;
1582 	admin->cache.sticky = ctr->sticky;
1583 	ri->vctrl->ops->cache_write(ri->ndev, admin, VCAP_SEL_COUNTER,
1584 				    ri->counter_id, 0);
1585 	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_WRITE,
1586 			       VCAP_SEL_COUNTER, ri->addr);
1587 	return 0;
1588 }
1589 
1590 /* Convert a chain id to a VCAP lookup index */
1591 int vcap_chain_id_to_lookup(struct vcap_admin *admin, int cur_cid)
1592 {
1593 	int lookup_first = admin->vinst * admin->lookups_per_instance;
1594 	int lookup_last = lookup_first + admin->lookups_per_instance;
1595 	int cid_next = admin->first_cid + VCAP_CID_LOOKUP_SIZE;
1596 	int cid = admin->first_cid;
1597 	int lookup;
1598 
1599 	for (lookup = lookup_first; lookup < lookup_last; ++lookup,
1600 	     cid += VCAP_CID_LOOKUP_SIZE, cid_next += VCAP_CID_LOOKUP_SIZE)
1601 		if (cur_cid >= cid && cur_cid < cid_next)
1602 			return lookup;
1603 	return 0;
1604 }
1605 EXPORT_SYMBOL_GPL(vcap_chain_id_to_lookup);
1606 
1607 /* Lookup a vcap instance using chain id */
1608 struct vcap_admin *vcap_find_admin(struct vcap_control *vctrl, int cid)
1609 {
1610 	struct vcap_admin *admin;
1611 
1612 	if (vcap_api_check(vctrl))
1613 		return NULL;
1614 
1615 	list_for_each_entry(admin, &vctrl->list, list) {
1616 		if (cid >= admin->first_cid && cid <= admin->last_cid)
1617 			return admin;
1618 	}
1619 	return NULL;
1620 }
1621 EXPORT_SYMBOL_GPL(vcap_find_admin);
1622 
1623 /* Is this the last admin instance ordered by chain id and direction */
1624 static bool vcap_admin_is_last(struct vcap_control *vctrl,
1625 			       struct vcap_admin *admin,
1626 			       bool ingress)
1627 {
1628 	struct vcap_admin *iter, *last = NULL;
1629 	int max_cid = 0;
1630 
1631 	list_for_each_entry(iter, &vctrl->list, list) {
1632 		if (iter->first_cid > max_cid &&
1633 		    iter->ingress == ingress) {
1634 			last = iter;
1635 			max_cid = iter->first_cid;
1636 		}
1637 	}
1638 	if (!last)
1639 		return false;
1640 
1641 	return admin == last;
1642 }
1643 
1644 /* Calculate the value used for chaining VCAP rules */
1645 int vcap_chain_offset(struct vcap_control *vctrl, int from_cid, int to_cid)
1646 {
1647 	int diff = to_cid - from_cid;
1648 
1649 	if (diff < 0) /* Wrong direction */
1650 		return diff;
1651 	to_cid %= VCAP_CID_LOOKUP_SIZE;
1652 	if (to_cid == 0)  /* Destination aligned to a lookup == no chaining */
1653 		return 0;
1654 	diff %= VCAP_CID_LOOKUP_SIZE;  /* Limit to a value within a lookup */
1655 	return diff;
1656 }
1657 EXPORT_SYMBOL_GPL(vcap_chain_offset);
1658 
1659 /* Is the next chain id in one of the following lookups
1660  * For now this does not support filters linked to other filters using
1661  * keys and actions. That will be added later.
1662  */
1663 bool vcap_is_next_lookup(struct vcap_control *vctrl, int src_cid, int dst_cid)
1664 {
1665 	struct vcap_admin *admin;
1666 	int next_cid;
1667 
1668 	if (vcap_api_check(vctrl))
1669 		return false;
1670 
1671 	/* The offset must be at least one lookup so round up one chain */
1672 	next_cid = roundup(src_cid + 1, VCAP_CID_LOOKUP_SIZE);
1673 
1674 	if (dst_cid < next_cid)
1675 		return false;
1676 
1677 	admin = vcap_find_admin(vctrl, dst_cid);
1678 	if (!admin)
1679 		return false;
1680 
1681 	return true;
1682 }
1683 EXPORT_SYMBOL_GPL(vcap_is_next_lookup);
1684 
1685 /* Check if there is room for a new rule */
1686 static int vcap_rule_space(struct vcap_admin *admin, int size)
1687 {
1688 	if (admin->last_used_addr - size < admin->first_valid_addr) {
1689 		pr_err("%s:%d: No room for rule size: %u, %u\n",
1690 		       __func__, __LINE__, size, admin->first_valid_addr);
1691 		return -ENOSPC;
1692 	}
1693 	return 0;
1694 }
1695 
1696 /* Add the keyset typefield to the list of rule keyfields */
1697 static int vcap_add_type_keyfield(struct vcap_rule *rule)
1698 {
1699 	struct vcap_rule_internal *ri = to_intrule(rule);
1700 	enum vcap_keyfield_set keyset = rule->keyset;
1701 	enum vcap_type vt = ri->admin->vtype;
1702 	const struct vcap_field *fields;
1703 	const struct vcap_set *kset;
1704 	int ret = -EINVAL;
1705 
1706 	kset = vcap_keyfieldset(ri->vctrl, vt, keyset);
1707 	if (!kset)
1708 		return ret;
1709 	if (kset->type_id == (u8)-1)  /* No type field is needed */
1710 		return 0;
1711 
1712 	fields = vcap_keyfields(ri->vctrl, vt, keyset);
1713 	if (!fields)
1714 		return -EINVAL;
1715 	if (fields[VCAP_KF_TYPE].width > 1) {
1716 		ret = vcap_rule_add_key_u32(rule, VCAP_KF_TYPE,
1717 					    kset->type_id, 0xff);
1718 	} else {
1719 		if (kset->type_id)
1720 			ret = vcap_rule_add_key_bit(rule, VCAP_KF_TYPE,
1721 						    VCAP_BIT_1);
1722 		else
1723 			ret = vcap_rule_add_key_bit(rule, VCAP_KF_TYPE,
1724 						    VCAP_BIT_0);
1725 	}
1726 	return 0;
1727 }
1728 
1729 /* Add the actionset typefield to the list of rule actionfields */
1730 static int vcap_add_type_actionfield(struct vcap_rule *rule)
1731 {
1732 	enum vcap_actionfield_set actionset = rule->actionset;
1733 	struct vcap_rule_internal *ri = to_intrule(rule);
1734 	enum vcap_type vt = ri->admin->vtype;
1735 	const struct vcap_field *fields;
1736 	const struct vcap_set *aset;
1737 	int ret = -EINVAL;
1738 
1739 	aset = vcap_actionfieldset(ri->vctrl, vt, actionset);
1740 	if (!aset)
1741 		return ret;
1742 	if (aset->type_id == (u8)-1)  /* No type field is needed */
1743 		return 0;
1744 
1745 	fields = vcap_actionfields(ri->vctrl, vt, actionset);
1746 	if (!fields)
1747 		return -EINVAL;
1748 	if (fields[VCAP_AF_TYPE].width > 1) {
1749 		ret = vcap_rule_add_action_u32(rule, VCAP_AF_TYPE,
1750 					       aset->type_id);
1751 	} else {
1752 		if (aset->type_id)
1753 			ret = vcap_rule_add_action_bit(rule, VCAP_AF_TYPE,
1754 						       VCAP_BIT_1);
1755 		else
1756 			ret = vcap_rule_add_action_bit(rule, VCAP_AF_TYPE,
1757 						       VCAP_BIT_0);
1758 	}
1759 	return ret;
1760 }
1761 
1762 /* Add a keyset to a keyset list */
1763 bool vcap_keyset_list_add(struct vcap_keyset_list *keysetlist,
1764 			  enum vcap_keyfield_set keyset)
1765 {
1766 	int idx;
1767 
1768 	if (keysetlist->cnt < keysetlist->max) {
1769 		/* Avoid duplicates */
1770 		for (idx = 0; idx < keysetlist->cnt; ++idx)
1771 			if (keysetlist->keysets[idx] == keyset)
1772 				return keysetlist->cnt < keysetlist->max;
1773 		keysetlist->keysets[keysetlist->cnt++] = keyset;
1774 	}
1775 	return keysetlist->cnt < keysetlist->max;
1776 }
1777 EXPORT_SYMBOL_GPL(vcap_keyset_list_add);
1778 
1779 /* Add a actionset to a actionset list */
1780 static bool vcap_actionset_list_add(struct vcap_actionset_list *actionsetlist,
1781 				    enum vcap_actionfield_set actionset)
1782 {
1783 	int idx;
1784 
1785 	if (actionsetlist->cnt < actionsetlist->max) {
1786 		/* Avoid duplicates */
1787 		for (idx = 0; idx < actionsetlist->cnt; ++idx)
1788 			if (actionsetlist->actionsets[idx] == actionset)
1789 				return actionsetlist->cnt < actionsetlist->max;
1790 		actionsetlist->actionsets[actionsetlist->cnt++] = actionset;
1791 	}
1792 	return actionsetlist->cnt < actionsetlist->max;
1793 }
1794 
1795 /* map keyset id to a string with the keyset name */
1796 const char *vcap_keyset_name(struct vcap_control *vctrl,
1797 			     enum vcap_keyfield_set keyset)
1798 {
1799 	return vctrl->stats->keyfield_set_names[keyset];
1800 }
1801 EXPORT_SYMBOL_GPL(vcap_keyset_name);
1802 
1803 /* map key field id to a string with the key name */
1804 const char *vcap_keyfield_name(struct vcap_control *vctrl,
1805 			       enum vcap_key_field key)
1806 {
1807 	return vctrl->stats->keyfield_names[key];
1808 }
1809 EXPORT_SYMBOL_GPL(vcap_keyfield_name);
1810 
1811 /* map actionset id to a string with the actionset name */
1812 const char *vcap_actionset_name(struct vcap_control *vctrl,
1813 				enum vcap_actionfield_set actionset)
1814 {
1815 	return vctrl->stats->actionfield_set_names[actionset];
1816 }
1817 
1818 /* map action field id to a string with the action name */
1819 const char *vcap_actionfield_name(struct vcap_control *vctrl,
1820 				  enum vcap_action_field action)
1821 {
1822 	return vctrl->stats->actionfield_names[action];
1823 }
1824 
1825 /* Return the keyfield that matches a key in a keyset */
1826 static const struct vcap_field *
1827 vcap_find_keyset_keyfield(struct vcap_control *vctrl,
1828 			  enum vcap_type vtype,
1829 			  enum vcap_keyfield_set keyset,
1830 			  enum vcap_key_field key)
1831 {
1832 	const struct vcap_field *fields;
1833 	int idx, count;
1834 
1835 	fields = vcap_keyfields(vctrl, vtype, keyset);
1836 	if (!fields)
1837 		return NULL;
1838 
1839 	/* Iterate the keyfields of the keyset */
1840 	count = vcap_keyfield_count(vctrl, vtype, keyset);
1841 	for (idx = 0; idx < count; ++idx) {
1842 		if (fields[idx].width == 0)
1843 			continue;
1844 
1845 		if (key == idx)
1846 			return &fields[idx];
1847 	}
1848 
1849 	return NULL;
1850 }
1851 
1852 /* Match a list of keys against the keysets available in a vcap type */
1853 static bool _vcap_rule_find_keysets(struct vcap_rule_internal *ri,
1854 				    struct vcap_keyset_list *matches)
1855 {
1856 	const struct vcap_client_keyfield *ckf;
1857 	int keyset, found, keycount, map_size;
1858 	const struct vcap_field **map;
1859 	enum vcap_type vtype;
1860 
1861 	vtype = ri->admin->vtype;
1862 	map = ri->vctrl->vcaps[vtype].keyfield_set_map;
1863 	map_size = ri->vctrl->vcaps[vtype].keyfield_set_size;
1864 
1865 	/* Get a count of the keyfields we want to match */
1866 	keycount = 0;
1867 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
1868 		++keycount;
1869 
1870 	matches->cnt = 0;
1871 	/* Iterate the keysets of the VCAP */
1872 	for (keyset = 0; keyset < map_size; ++keyset) {
1873 		if (!map[keyset])
1874 			continue;
1875 
1876 		/* Iterate the keys in the rule */
1877 		found = 0;
1878 		list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
1879 			if (vcap_find_keyset_keyfield(ri->vctrl, vtype,
1880 						      keyset, ckf->ctrl.key))
1881 				++found;
1882 
1883 		/* Save the keyset if all keyfields were found */
1884 		if (found == keycount)
1885 			if (!vcap_keyset_list_add(matches, keyset))
1886 				/* bail out when the quota is filled */
1887 				break;
1888 	}
1889 
1890 	return matches->cnt > 0;
1891 }
1892 
1893 /* Match a list of keys against the keysets available in a vcap type */
1894 bool vcap_rule_find_keysets(struct vcap_rule *rule,
1895 			    struct vcap_keyset_list *matches)
1896 {
1897 	struct vcap_rule_internal *ri = to_intrule(rule);
1898 
1899 	return _vcap_rule_find_keysets(ri, matches);
1900 }
1901 EXPORT_SYMBOL_GPL(vcap_rule_find_keysets);
1902 
1903 /* Return the actionfield that matches a action in a actionset */
1904 static const struct vcap_field *
1905 vcap_find_actionset_actionfield(struct vcap_control *vctrl,
1906 				enum vcap_type vtype,
1907 				enum vcap_actionfield_set actionset,
1908 				enum vcap_action_field action)
1909 {
1910 	const struct vcap_field *fields;
1911 	int idx, count;
1912 
1913 	fields = vcap_actionfields(vctrl, vtype, actionset);
1914 	if (!fields)
1915 		return NULL;
1916 
1917 	/* Iterate the actionfields of the actionset */
1918 	count = vcap_actionfield_count(vctrl, vtype, actionset);
1919 	for (idx = 0; idx < count; ++idx) {
1920 		if (fields[idx].width == 0)
1921 			continue;
1922 
1923 		if (action == idx)
1924 			return &fields[idx];
1925 	}
1926 
1927 	return NULL;
1928 }
1929 
1930 /* Match a list of actions against the actionsets available in a vcap type */
1931 static bool vcap_rule_find_actionsets(struct vcap_rule_internal *ri,
1932 				      struct vcap_actionset_list *matches)
1933 {
1934 	int actionset, found, actioncount, map_size;
1935 	const struct vcap_client_actionfield *ckf;
1936 	const struct vcap_field **map;
1937 	enum vcap_type vtype;
1938 
1939 	vtype = ri->admin->vtype;
1940 	map = ri->vctrl->vcaps[vtype].actionfield_set_map;
1941 	map_size = ri->vctrl->vcaps[vtype].actionfield_set_size;
1942 
1943 	/* Get a count of the actionfields we want to match */
1944 	actioncount = 0;
1945 	list_for_each_entry(ckf, &ri->data.actionfields, ctrl.list)
1946 		++actioncount;
1947 
1948 	matches->cnt = 0;
1949 	/* Iterate the actionsets of the VCAP */
1950 	for (actionset = 0; actionset < map_size; ++actionset) {
1951 		if (!map[actionset])
1952 			continue;
1953 
1954 		/* Iterate the actions in the rule */
1955 		found = 0;
1956 		list_for_each_entry(ckf, &ri->data.actionfields, ctrl.list)
1957 			if (vcap_find_actionset_actionfield(ri->vctrl, vtype,
1958 							    actionset,
1959 							    ckf->ctrl.action))
1960 				++found;
1961 
1962 		/* Save the actionset if all actionfields were found */
1963 		if (found == actioncount)
1964 			if (!vcap_actionset_list_add(matches, actionset))
1965 				/* bail out when the quota is filled */
1966 				break;
1967 	}
1968 
1969 	return matches->cnt > 0;
1970 }
1971 
1972 /* Validate a rule with respect to available port keys */
1973 int vcap_val_rule(struct vcap_rule *rule, u16 l3_proto)
1974 {
1975 	struct vcap_rule_internal *ri = to_intrule(rule);
1976 	struct vcap_keyset_list matches = {};
1977 	enum vcap_keyfield_set keysets[10];
1978 	int ret;
1979 
1980 	ret = vcap_api_check(ri->vctrl);
1981 	if (ret)
1982 		return ret;
1983 	if (!ri->admin) {
1984 		ri->data.exterr = VCAP_ERR_NO_ADMIN;
1985 		return -EINVAL;
1986 	}
1987 	if (!ri->ndev) {
1988 		ri->data.exterr = VCAP_ERR_NO_NETDEV;
1989 		return -EINVAL;
1990 	}
1991 
1992 	matches.keysets = keysets;
1993 	matches.max = ARRAY_SIZE(keysets);
1994 	if (ri->data.keyset == VCAP_KFS_NO_VALUE) {
1995 		/* Iterate over rule keyfields and select keysets that fits */
1996 		if (!_vcap_rule_find_keysets(ri, &matches)) {
1997 			ri->data.exterr = VCAP_ERR_NO_KEYSET_MATCH;
1998 			return -EINVAL;
1999 		}
2000 	} else {
2001 		/* prepare for keyset validation */
2002 		keysets[0] = ri->data.keyset;
2003 		matches.cnt = 1;
2004 	}
2005 
2006 	/* Pick a keyset that is supported in the port lookups */
2007 	ret = ri->vctrl->ops->validate_keyset(ri->ndev, ri->admin, rule,
2008 					      &matches, l3_proto);
2009 	if (ret < 0) {
2010 		pr_err("%s:%d: keyset validation failed: %d\n",
2011 		       __func__, __LINE__, ret);
2012 		ri->data.exterr = VCAP_ERR_NO_PORT_KEYSET_MATCH;
2013 		return ret;
2014 	}
2015 	/* use the keyset that is supported in the port lookups */
2016 	ret = vcap_set_rule_set_keyset(rule, ret);
2017 	if (ret < 0) {
2018 		pr_err("%s:%d: keyset was not updated: %d\n",
2019 		       __func__, __LINE__, ret);
2020 		return ret;
2021 	}
2022 	if (ri->data.actionset == VCAP_AFS_NO_VALUE) {
2023 		struct vcap_actionset_list matches = {};
2024 		enum vcap_actionfield_set actionsets[10];
2025 
2026 		matches.actionsets = actionsets;
2027 		matches.max = ARRAY_SIZE(actionsets);
2028 
2029 		/* Find an actionset that fits the rule actions */
2030 		if (!vcap_rule_find_actionsets(ri, &matches)) {
2031 			ri->data.exterr = VCAP_ERR_NO_ACTIONSET_MATCH;
2032 			return -EINVAL;
2033 		}
2034 		ret = vcap_set_rule_set_actionset(rule, actionsets[0]);
2035 		if (ret < 0) {
2036 			pr_err("%s:%d: actionset was not updated: %d\n",
2037 			       __func__, __LINE__, ret);
2038 			return ret;
2039 		}
2040 	}
2041 	vcap_add_type_keyfield(rule);
2042 	vcap_add_type_actionfield(rule);
2043 	/* Add default fields to this rule */
2044 	ri->vctrl->ops->add_default_fields(ri->ndev, ri->admin, rule);
2045 
2046 	/* Rule size is the maximum of the entry and action subword count */
2047 	ri->size = max(ri->keyset_sw, ri->actionset_sw);
2048 
2049 	/* Finally check if there is room for the rule in the VCAP */
2050 	return vcap_rule_space(ri->admin, ri->size);
2051 }
2052 EXPORT_SYMBOL_GPL(vcap_val_rule);
2053 
2054 /* Entries are sorted with increasing values of sort_key.
2055  * I.e. Lowest numerical sort_key is first in list.
2056  * In order to locate largest keys first in list we negate the key size with
2057  * (max_size - size).
2058  */
2059 static u32 vcap_sort_key(u32 max_size, u32 size, u8 user, u16 prio)
2060 {
2061 	return ((max_size - size) << 24) | (user << 16) | prio;
2062 }
2063 
2064 /* calculate the address of the next rule after this (lower address and prio) */
2065 static u32 vcap_next_rule_addr(u32 addr, struct vcap_rule_internal *ri)
2066 {
2067 	return ((addr - ri->size) /  ri->size) * ri->size;
2068 }
2069 
2070 /* Assign a unique rule id and autogenerate one if id == 0 */
2071 static u32 vcap_set_rule_id(struct vcap_rule_internal *ri)
2072 {
2073 	if (ri->data.id != 0)
2074 		return ri->data.id;
2075 
2076 	for (u32 next_id = 1; next_id < ~0; ++next_id) {
2077 		if (!vcap_rule_exists(ri->vctrl, next_id)) {
2078 			ri->data.id = next_id;
2079 			break;
2080 		}
2081 	}
2082 	return ri->data.id;
2083 }
2084 
2085 static int vcap_insert_rule(struct vcap_rule_internal *ri,
2086 			    struct vcap_rule_move *move)
2087 {
2088 	int sw_count = ri->vctrl->vcaps[ri->admin->vtype].sw_count;
2089 	struct vcap_rule_internal *duprule, *iter, *elem = NULL;
2090 	struct vcap_admin *admin = ri->admin;
2091 	u32 addr;
2092 
2093 	ri->sort_key = vcap_sort_key(sw_count, ri->size, ri->data.user,
2094 				     ri->data.priority);
2095 
2096 	/* Insert the new rule in the list of rule based on the sort key
2097 	 * If the rule needs to be  inserted between existing rules then move
2098 	 * these rules to make room for the new rule and update their start
2099 	 * address.
2100 	 */
2101 	list_for_each_entry(iter, &admin->rules, list) {
2102 		if (ri->sort_key < iter->sort_key) {
2103 			elem = iter;
2104 			break;
2105 		}
2106 	}
2107 
2108 	if (!elem) {
2109 		ri->addr = vcap_next_rule_addr(admin->last_used_addr, ri);
2110 		admin->last_used_addr = ri->addr;
2111 
2112 		/* Add a copy of the rule to the VCAP list */
2113 		duprule = vcap_dup_rule(ri, ri->state == VCAP_RS_DISABLED);
2114 		if (IS_ERR(duprule))
2115 			return PTR_ERR(duprule);
2116 
2117 		list_add_tail(&duprule->list, &admin->rules);
2118 		return 0;
2119 	}
2120 
2121 	/* Reuse the space of the current rule */
2122 	addr = elem->addr + elem->size;
2123 	ri->addr = vcap_next_rule_addr(addr, ri);
2124 	addr = ri->addr;
2125 
2126 	/* Add a copy of the rule to the VCAP list */
2127 	duprule = vcap_dup_rule(ri, ri->state == VCAP_RS_DISABLED);
2128 	if (IS_ERR(duprule))
2129 		return PTR_ERR(duprule);
2130 
2131 	/* Add before the current entry */
2132 	list_add_tail(&duprule->list, &elem->list);
2133 
2134 	/* Update the current rule */
2135 	elem->addr = vcap_next_rule_addr(addr, elem);
2136 	addr = elem->addr;
2137 
2138 	/* Update the address in the remaining rules in the list */
2139 	list_for_each_entry_continue(elem, &admin->rules, list) {
2140 		elem->addr = vcap_next_rule_addr(addr, elem);
2141 		addr = elem->addr;
2142 	}
2143 
2144 	/* Update the move info */
2145 	move->addr = admin->last_used_addr;
2146 	move->count = ri->addr - addr;
2147 	move->offset = admin->last_used_addr - addr;
2148 	admin->last_used_addr = addr;
2149 	return 0;
2150 }
2151 
2152 static void vcap_move_rules(struct vcap_rule_internal *ri,
2153 			    struct vcap_rule_move *move)
2154 {
2155 	ri->vctrl->ops->move(ri->ndev, ri->admin, move->addr,
2156 			 move->offset, move->count);
2157 }
2158 
2159 /* Check if the chain is already used to enable a VCAP lookup for this port */
2160 static bool vcap_is_chain_used(struct vcap_control *vctrl,
2161 			       struct net_device *ndev, int src_cid)
2162 {
2163 	struct vcap_enabled_port *eport;
2164 	struct vcap_admin *admin;
2165 
2166 	list_for_each_entry(admin, &vctrl->list, list)
2167 		list_for_each_entry(eport, &admin->enabled, list)
2168 			if (eport->src_cid == src_cid && eport->ndev == ndev)
2169 				return true;
2170 
2171 	return false;
2172 }
2173 
2174 /* Fetch the next chain in the enabled list for the port */
2175 static int vcap_get_next_chain(struct vcap_control *vctrl,
2176 			       struct net_device *ndev,
2177 			       int dst_cid)
2178 {
2179 	struct vcap_enabled_port *eport;
2180 	struct vcap_admin *admin;
2181 
2182 	list_for_each_entry(admin, &vctrl->list, list) {
2183 		list_for_each_entry(eport, &admin->enabled, list) {
2184 			if (eport->ndev != ndev)
2185 				continue;
2186 			if (eport->src_cid == dst_cid)
2187 				return eport->dst_cid;
2188 		}
2189 	}
2190 
2191 	return 0;
2192 }
2193 
2194 static bool vcap_path_exist(struct vcap_control *vctrl, struct net_device *ndev,
2195 			    int dst_cid)
2196 {
2197 	int cid = rounddown(dst_cid, VCAP_CID_LOOKUP_SIZE);
2198 	struct vcap_enabled_port *eport = NULL;
2199 	struct vcap_enabled_port *elem;
2200 	struct vcap_admin *admin;
2201 	int tmp;
2202 
2203 	if (cid == 0) /* Chain zero is always available */
2204 		return true;
2205 
2206 	/* Find first entry that starts from chain 0*/
2207 	list_for_each_entry(admin, &vctrl->list, list) {
2208 		list_for_each_entry(elem, &admin->enabled, list) {
2209 			if (elem->src_cid == 0 && elem->ndev == ndev) {
2210 				eport = elem;
2211 				break;
2212 			}
2213 		}
2214 		if (eport)
2215 			break;
2216 	}
2217 
2218 	if (!eport)
2219 		return false;
2220 
2221 	tmp = eport->dst_cid;
2222 	while (tmp != cid && tmp != 0)
2223 		tmp = vcap_get_next_chain(vctrl, ndev, tmp);
2224 
2225 	return !!tmp;
2226 }
2227 
2228 /* Internal clients can always store their rules in HW
2229  * External clients can store their rules if the chain is enabled all
2230  * the way from chain 0, otherwise the rule will be cached until
2231  * the chain is enabled.
2232  */
2233 static void vcap_rule_set_state(struct vcap_rule_internal *ri)
2234 {
2235 	if (ri->data.user <= VCAP_USER_QOS)
2236 		ri->state = VCAP_RS_PERMANENT;
2237 	else if (vcap_path_exist(ri->vctrl, ri->ndev, ri->data.vcap_chain_id))
2238 		ri->state = VCAP_RS_ENABLED;
2239 	else
2240 		ri->state = VCAP_RS_DISABLED;
2241 }
2242 
2243 /* Encode and write a validated rule to the VCAP */
2244 int vcap_add_rule(struct vcap_rule *rule)
2245 {
2246 	struct vcap_rule_internal *ri = to_intrule(rule);
2247 	struct vcap_rule_move move = {0};
2248 	struct vcap_counter ctr = {0};
2249 	int ret;
2250 
2251 	ret = vcap_api_check(ri->vctrl);
2252 	if (ret)
2253 		return ret;
2254 	/* Insert the new rule in the list of vcap rules */
2255 	mutex_lock(&ri->admin->lock);
2256 
2257 	vcap_rule_set_state(ri);
2258 	ret = vcap_insert_rule(ri, &move);
2259 	if (ret < 0) {
2260 		pr_err("%s:%d: could not insert rule in vcap list: %d\n",
2261 		       __func__, __LINE__, ret);
2262 		goto out;
2263 	}
2264 	if (move.count > 0)
2265 		vcap_move_rules(ri, &move);
2266 
2267 	/* Set the counter to zero */
2268 	ret = vcap_write_counter(ri, &ctr);
2269 	if (ret)
2270 		goto out;
2271 
2272 	if (ri->state == VCAP_RS_DISABLED) {
2273 		/* Erase the rule area */
2274 		ri->vctrl->ops->init(ri->ndev, ri->admin, ri->addr, ri->size);
2275 		goto out;
2276 	}
2277 
2278 	vcap_erase_cache(ri);
2279 	ret = vcap_encode_rule(ri);
2280 	if (ret) {
2281 		pr_err("%s:%d: rule encoding error: %d\n", __func__, __LINE__, ret);
2282 		goto out;
2283 	}
2284 
2285 	ret = vcap_write_rule(ri);
2286 	if (ret) {
2287 		pr_err("%s:%d: rule write error: %d\n", __func__, __LINE__, ret);
2288 		goto out;
2289 	}
2290 out:
2291 	mutex_unlock(&ri->admin->lock);
2292 	return ret;
2293 }
2294 EXPORT_SYMBOL_GPL(vcap_add_rule);
2295 
2296 /* Allocate a new rule with the provided arguments */
2297 struct vcap_rule *vcap_alloc_rule(struct vcap_control *vctrl,
2298 				  struct net_device *ndev, int vcap_chain_id,
2299 				  enum vcap_user user, u16 priority,
2300 				  u32 id)
2301 {
2302 	struct vcap_rule_internal *ri;
2303 	struct vcap_admin *admin;
2304 	int err, maxsize;
2305 
2306 	err = vcap_api_check(vctrl);
2307 	if (err)
2308 		return ERR_PTR(err);
2309 	if (!ndev)
2310 		return ERR_PTR(-ENODEV);
2311 	/* Get the VCAP instance */
2312 	admin = vcap_find_admin(vctrl, vcap_chain_id);
2313 	if (!admin)
2314 		return ERR_PTR(-ENOENT);
2315 	/* Sanity check that this VCAP is supported on this platform */
2316 	if (vctrl->vcaps[admin->vtype].rows == 0)
2317 		return ERR_PTR(-EINVAL);
2318 
2319 	mutex_lock(&admin->lock);
2320 	/* Check if a rule with this id already exists */
2321 	if (vcap_rule_exists(vctrl, id)) {
2322 		err = -EINVAL;
2323 		goto out_unlock;
2324 	}
2325 
2326 	/* Check if there is room for the rule in the block(s) of the VCAP */
2327 	maxsize = vctrl->vcaps[admin->vtype].sw_count; /* worst case rule size */
2328 	if (vcap_rule_space(admin, maxsize)) {
2329 		err = -ENOSPC;
2330 		goto out_unlock;
2331 	}
2332 
2333 	/* Create a container for the rule and return it */
2334 	ri = kzalloc(sizeof(*ri), GFP_KERNEL);
2335 	if (!ri) {
2336 		err = -ENOMEM;
2337 		goto out_unlock;
2338 	}
2339 
2340 	ri->data.vcap_chain_id = vcap_chain_id;
2341 	ri->data.user = user;
2342 	ri->data.priority = priority;
2343 	ri->data.id = id;
2344 	ri->data.keyset = VCAP_KFS_NO_VALUE;
2345 	ri->data.actionset = VCAP_AFS_NO_VALUE;
2346 	INIT_LIST_HEAD(&ri->list);
2347 	INIT_LIST_HEAD(&ri->data.keyfields);
2348 	INIT_LIST_HEAD(&ri->data.actionfields);
2349 	ri->ndev = ndev;
2350 	ri->admin = admin; /* refer to the vcap instance */
2351 	ri->vctrl = vctrl; /* refer to the client */
2352 
2353 	if (vcap_set_rule_id(ri) == 0) {
2354 		err = -EINVAL;
2355 		goto out_free;
2356 	}
2357 
2358 	mutex_unlock(&admin->lock);
2359 	return (struct vcap_rule *)ri;
2360 
2361 out_free:
2362 	kfree(ri);
2363 out_unlock:
2364 	mutex_unlock(&admin->lock);
2365 	return ERR_PTR(err);
2366 
2367 }
2368 EXPORT_SYMBOL_GPL(vcap_alloc_rule);
2369 
2370 /* Free mem of a rule owned by client after the rule as been added to the VCAP */
2371 void vcap_free_rule(struct vcap_rule *rule)
2372 {
2373 	struct vcap_rule_internal *ri = to_intrule(rule);
2374 	struct vcap_client_actionfield *caf, *next_caf;
2375 	struct vcap_client_keyfield *ckf, *next_ckf;
2376 
2377 	/* Deallocate the list of keys and actions */
2378 	list_for_each_entry_safe(ckf, next_ckf, &ri->data.keyfields, ctrl.list) {
2379 		list_del(&ckf->ctrl.list);
2380 		kfree(ckf);
2381 	}
2382 	list_for_each_entry_safe(caf, next_caf, &ri->data.actionfields, ctrl.list) {
2383 		list_del(&caf->ctrl.list);
2384 		kfree(caf);
2385 	}
2386 	/* Deallocate the rule */
2387 	kfree(rule);
2388 }
2389 EXPORT_SYMBOL_GPL(vcap_free_rule);
2390 
2391 /* Decode a rule from the VCAP cache and return a copy */
2392 struct vcap_rule *vcap_decode_rule(struct vcap_rule_internal *elem)
2393 {
2394 	struct vcap_rule_internal *ri;
2395 	int err;
2396 
2397 	ri = vcap_dup_rule(elem, elem->state == VCAP_RS_DISABLED);
2398 	if (IS_ERR(ri))
2399 		return ERR_CAST(ri);
2400 
2401 	if (ri->state == VCAP_RS_DISABLED)
2402 		goto out;
2403 
2404 	err = vcap_read_rule(ri);
2405 	if (err)
2406 		return ERR_PTR(err);
2407 
2408 	err = vcap_decode_keyset(ri);
2409 	if (err)
2410 		return ERR_PTR(err);
2411 
2412 	err = vcap_decode_actionset(ri);
2413 	if (err)
2414 		return ERR_PTR(err);
2415 
2416 out:
2417 	return &ri->data;
2418 }
2419 
2420 struct vcap_rule *vcap_get_rule(struct vcap_control *vctrl, u32 id)
2421 {
2422 	struct vcap_rule_internal *elem;
2423 	struct vcap_rule *rule;
2424 	int err;
2425 
2426 	err = vcap_api_check(vctrl);
2427 	if (err)
2428 		return ERR_PTR(err);
2429 
2430 	elem = vcap_get_locked_rule(vctrl, id);
2431 	if (!elem)
2432 		return ERR_PTR(-ENOENT);
2433 
2434 	rule = vcap_decode_rule(elem);
2435 	mutex_unlock(&elem->admin->lock);
2436 	return rule;
2437 }
2438 EXPORT_SYMBOL_GPL(vcap_get_rule);
2439 
2440 /* Update existing rule */
2441 int vcap_mod_rule(struct vcap_rule *rule)
2442 {
2443 	struct vcap_rule_internal *ri = to_intrule(rule);
2444 	struct vcap_counter ctr;
2445 	int err;
2446 
2447 	err = vcap_api_check(ri->vctrl);
2448 	if (err)
2449 		return err;
2450 
2451 	if (!vcap_get_locked_rule(ri->vctrl, ri->data.id))
2452 		return -ENOENT;
2453 
2454 	vcap_rule_set_state(ri);
2455 	if (ri->state == VCAP_RS_DISABLED)
2456 		goto out;
2457 
2458 	/* Encode the bitstreams to the VCAP cache */
2459 	vcap_erase_cache(ri);
2460 	err = vcap_encode_rule(ri);
2461 	if (err)
2462 		goto out;
2463 
2464 	err = vcap_write_rule(ri);
2465 	if (err)
2466 		goto out;
2467 
2468 	memset(&ctr, 0, sizeof(ctr));
2469 	err =  vcap_write_counter(ri, &ctr);
2470 
2471 out:
2472 	mutex_unlock(&ri->admin->lock);
2473 	return err;
2474 }
2475 EXPORT_SYMBOL_GPL(vcap_mod_rule);
2476 
2477 /* Return the alignment offset for a new rule address */
2478 static int vcap_valid_rule_move(struct vcap_rule_internal *el, int offset)
2479 {
2480 	return (el->addr + offset) % el->size;
2481 }
2482 
2483 /* Update the rule address with an offset */
2484 static void vcap_adjust_rule_addr(struct vcap_rule_internal *el, int offset)
2485 {
2486 	el->addr += offset;
2487 }
2488 
2489 /* Rules needs to be moved to fill the gap of the deleted rule */
2490 static int vcap_fill_rule_gap(struct vcap_rule_internal *ri)
2491 {
2492 	struct vcap_admin *admin = ri->admin;
2493 	struct vcap_rule_internal *elem;
2494 	struct vcap_rule_move move;
2495 	int gap = 0, offset = 0;
2496 
2497 	/* If the first rule is deleted: Move other rules to the top */
2498 	if (list_is_first(&ri->list, &admin->rules))
2499 		offset = admin->last_valid_addr + 1 - ri->addr - ri->size;
2500 
2501 	/* Locate gaps between odd size rules and adjust the move */
2502 	elem = ri;
2503 	list_for_each_entry_continue(elem, &admin->rules, list)
2504 		gap += vcap_valid_rule_move(elem, ri->size);
2505 
2506 	/* Update the address in the remaining rules in the list */
2507 	elem = ri;
2508 	list_for_each_entry_continue(elem, &admin->rules, list)
2509 		vcap_adjust_rule_addr(elem, ri->size + gap + offset);
2510 
2511 	/* Update the move info */
2512 	move.addr = admin->last_used_addr;
2513 	move.count = ri->addr - admin->last_used_addr - gap;
2514 	move.offset = -(ri->size + gap + offset);
2515 
2516 	/* Do the actual move operation */
2517 	vcap_move_rules(ri, &move);
2518 
2519 	return gap + offset;
2520 }
2521 
2522 /* Delete rule in a VCAP instance */
2523 int vcap_del_rule(struct vcap_control *vctrl, struct net_device *ndev, u32 id)
2524 {
2525 	struct vcap_rule_internal *ri, *elem;
2526 	struct vcap_admin *admin;
2527 	int gap = 0, err;
2528 
2529 	/* This will later also handle rule moving */
2530 	if (!ndev)
2531 		return -ENODEV;
2532 	err = vcap_api_check(vctrl);
2533 	if (err)
2534 		return err;
2535 	/* Look for the rule id in all vcaps */
2536 	ri = vcap_get_locked_rule(vctrl, id);
2537 	if (!ri)
2538 		return -ENOENT;
2539 
2540 	admin = ri->admin;
2541 
2542 	if (ri->addr > admin->last_used_addr)
2543 		gap = vcap_fill_rule_gap(ri);
2544 
2545 	/* Delete the rule from the list of rules and the cache */
2546 	list_del(&ri->list);
2547 	vctrl->ops->init(ndev, admin, admin->last_used_addr, ri->size + gap);
2548 	vcap_free_rule(&ri->data);
2549 
2550 	/* Update the last used address, set to default when no rules */
2551 	if (list_empty(&admin->rules)) {
2552 		admin->last_used_addr = admin->last_valid_addr + 1;
2553 	} else {
2554 		elem = list_last_entry(&admin->rules, struct vcap_rule_internal,
2555 				       list);
2556 		admin->last_used_addr = elem->addr;
2557 	}
2558 
2559 	mutex_unlock(&admin->lock);
2560 	return err;
2561 }
2562 EXPORT_SYMBOL_GPL(vcap_del_rule);
2563 
2564 /* Delete all rules in the VCAP instance */
2565 int vcap_del_rules(struct vcap_control *vctrl, struct vcap_admin *admin)
2566 {
2567 	struct vcap_enabled_port *eport, *next_eport;
2568 	struct vcap_rule_internal *ri, *next_ri;
2569 	int ret = vcap_api_check(vctrl);
2570 
2571 	if (ret)
2572 		return ret;
2573 
2574 	mutex_lock(&admin->lock);
2575 	list_for_each_entry_safe(ri, next_ri, &admin->rules, list) {
2576 		vctrl->ops->init(ri->ndev, admin, ri->addr, ri->size);
2577 		list_del(&ri->list);
2578 		vcap_free_rule(&ri->data);
2579 	}
2580 	admin->last_used_addr = admin->last_valid_addr;
2581 
2582 	/* Remove list of enabled ports */
2583 	list_for_each_entry_safe(eport, next_eport, &admin->enabled, list) {
2584 		list_del(&eport->list);
2585 		kfree(eport);
2586 	}
2587 	mutex_unlock(&admin->lock);
2588 
2589 	return 0;
2590 }
2591 EXPORT_SYMBOL_GPL(vcap_del_rules);
2592 
2593 /* Find a client key field in a rule */
2594 static struct vcap_client_keyfield *
2595 vcap_find_keyfield(struct vcap_rule *rule, enum vcap_key_field key)
2596 {
2597 	struct vcap_rule_internal *ri = to_intrule(rule);
2598 	struct vcap_client_keyfield *ckf;
2599 
2600 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
2601 		if (ckf->ctrl.key == key)
2602 			return ckf;
2603 	return NULL;
2604 }
2605 
2606 /* Find information on a key field in a rule */
2607 const struct vcap_field *vcap_lookup_keyfield(struct vcap_rule *rule,
2608 					      enum vcap_key_field key)
2609 {
2610 	struct vcap_rule_internal *ri = to_intrule(rule);
2611 	enum vcap_keyfield_set keyset = rule->keyset;
2612 	enum vcap_type vt = ri->admin->vtype;
2613 	const struct vcap_field *fields;
2614 
2615 	if (keyset == VCAP_KFS_NO_VALUE)
2616 		return NULL;
2617 	fields = vcap_keyfields(ri->vctrl, vt, keyset);
2618 	if (!fields)
2619 		return NULL;
2620 	return &fields[key];
2621 }
2622 EXPORT_SYMBOL_GPL(vcap_lookup_keyfield);
2623 
2624 /* Check if the keyfield is already in the rule */
2625 static bool vcap_keyfield_unique(struct vcap_rule *rule,
2626 				 enum vcap_key_field key)
2627 {
2628 	struct vcap_rule_internal *ri = to_intrule(rule);
2629 	const struct vcap_client_keyfield *ckf;
2630 
2631 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list)
2632 		if (ckf->ctrl.key == key)
2633 			return false;
2634 	return true;
2635 }
2636 
2637 /* Check if the keyfield is in the keyset */
2638 static bool vcap_keyfield_match_keyset(struct vcap_rule *rule,
2639 				       enum vcap_key_field key)
2640 {
2641 	struct vcap_rule_internal *ri = to_intrule(rule);
2642 	enum vcap_keyfield_set keyset = rule->keyset;
2643 	enum vcap_type vt = ri->admin->vtype;
2644 	const struct vcap_field *fields;
2645 
2646 	/* the field is accepted if the rule has no keyset yet */
2647 	if (keyset == VCAP_KFS_NO_VALUE)
2648 		return true;
2649 	fields = vcap_keyfields(ri->vctrl, vt, keyset);
2650 	if (!fields)
2651 		return false;
2652 	/* if there is a width there is a way */
2653 	return fields[key].width > 0;
2654 }
2655 
2656 static int vcap_rule_add_key(struct vcap_rule *rule,
2657 			     enum vcap_key_field key,
2658 			     enum vcap_field_type ftype,
2659 			     struct vcap_client_keyfield_data *data)
2660 {
2661 	struct vcap_rule_internal *ri = to_intrule(rule);
2662 	struct vcap_client_keyfield *field;
2663 
2664 	if (!vcap_keyfield_unique(rule, key)) {
2665 		pr_warn("%s:%d: keyfield %s is already in the rule\n",
2666 			__func__, __LINE__,
2667 			vcap_keyfield_name(ri->vctrl, key));
2668 		return -EINVAL;
2669 	}
2670 
2671 	if (!vcap_keyfield_match_keyset(rule, key)) {
2672 		pr_err("%s:%d: keyfield %s does not belong in the rule keyset\n",
2673 		       __func__, __LINE__,
2674 		       vcap_keyfield_name(ri->vctrl, key));
2675 		return -EINVAL;
2676 	}
2677 
2678 	field = kzalloc(sizeof(*field), GFP_KERNEL);
2679 	if (!field)
2680 		return -ENOMEM;
2681 	memcpy(&field->data, data, sizeof(field->data));
2682 	field->ctrl.key = key;
2683 	field->ctrl.type = ftype;
2684 	list_add_tail(&field->ctrl.list, &rule->keyfields);
2685 	return 0;
2686 }
2687 
2688 static void vcap_rule_set_key_bitsize(struct vcap_u1_key *u1, enum vcap_bit val)
2689 {
2690 	switch (val) {
2691 	case VCAP_BIT_0:
2692 		u1->value = 0;
2693 		u1->mask = 1;
2694 		break;
2695 	case VCAP_BIT_1:
2696 		u1->value = 1;
2697 		u1->mask = 1;
2698 		break;
2699 	case VCAP_BIT_ANY:
2700 		u1->value = 0;
2701 		u1->mask = 0;
2702 		break;
2703 	}
2704 }
2705 
2706 /* Add a bit key with value and mask to the rule */
2707 int vcap_rule_add_key_bit(struct vcap_rule *rule, enum vcap_key_field key,
2708 			  enum vcap_bit val)
2709 {
2710 	struct vcap_client_keyfield_data data;
2711 
2712 	vcap_rule_set_key_bitsize(&data.u1, val);
2713 	return vcap_rule_add_key(rule, key, VCAP_FIELD_BIT, &data);
2714 }
2715 EXPORT_SYMBOL_GPL(vcap_rule_add_key_bit);
2716 
2717 /* Add a 32 bit key field with value and mask to the rule */
2718 int vcap_rule_add_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
2719 			  u32 value, u32 mask)
2720 {
2721 	struct vcap_client_keyfield_data data;
2722 
2723 	data.u32.value = value;
2724 	data.u32.mask = mask;
2725 	return vcap_rule_add_key(rule, key, VCAP_FIELD_U32, &data);
2726 }
2727 EXPORT_SYMBOL_GPL(vcap_rule_add_key_u32);
2728 
2729 /* Add a 48 bit key with value and mask to the rule */
2730 int vcap_rule_add_key_u48(struct vcap_rule *rule, enum vcap_key_field key,
2731 			  struct vcap_u48_key *fieldval)
2732 {
2733 	struct vcap_client_keyfield_data data;
2734 
2735 	memcpy(&data.u48, fieldval, sizeof(data.u48));
2736 	return vcap_rule_add_key(rule, key, VCAP_FIELD_U48, &data);
2737 }
2738 EXPORT_SYMBOL_GPL(vcap_rule_add_key_u48);
2739 
2740 /* Add a 72 bit key with value and mask to the rule */
2741 int vcap_rule_add_key_u72(struct vcap_rule *rule, enum vcap_key_field key,
2742 			  struct vcap_u72_key *fieldval)
2743 {
2744 	struct vcap_client_keyfield_data data;
2745 
2746 	memcpy(&data.u72, fieldval, sizeof(data.u72));
2747 	return vcap_rule_add_key(rule, key, VCAP_FIELD_U72, &data);
2748 }
2749 EXPORT_SYMBOL_GPL(vcap_rule_add_key_u72);
2750 
2751 /* Add a 128 bit key with value and mask to the rule */
2752 int vcap_rule_add_key_u128(struct vcap_rule *rule, enum vcap_key_field key,
2753 			   struct vcap_u128_key *fieldval)
2754 {
2755 	struct vcap_client_keyfield_data data;
2756 
2757 	memcpy(&data.u128, fieldval, sizeof(data.u128));
2758 	return vcap_rule_add_key(rule, key, VCAP_FIELD_U128, &data);
2759 }
2760 EXPORT_SYMBOL_GPL(vcap_rule_add_key_u128);
2761 
2762 int vcap_rule_get_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
2763 			  u32 *value, u32 *mask)
2764 {
2765 	struct vcap_client_keyfield *ckf;
2766 
2767 	ckf = vcap_find_keyfield(rule, key);
2768 	if (!ckf)
2769 		return -ENOENT;
2770 
2771 	*value = ckf->data.u32.value;
2772 	*mask = ckf->data.u32.mask;
2773 
2774 	return 0;
2775 }
2776 EXPORT_SYMBOL_GPL(vcap_rule_get_key_u32);
2777 
2778 /* Find a client action field in a rule */
2779 struct vcap_client_actionfield *
2780 vcap_find_actionfield(struct vcap_rule *rule, enum vcap_action_field act)
2781 {
2782 	struct vcap_rule_internal *ri = (struct vcap_rule_internal *)rule;
2783 	struct vcap_client_actionfield *caf;
2784 
2785 	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list)
2786 		if (caf->ctrl.action == act)
2787 			return caf;
2788 	return NULL;
2789 }
2790 EXPORT_SYMBOL_GPL(vcap_find_actionfield);
2791 
2792 /* Check if the actionfield is already in the rule */
2793 static bool vcap_actionfield_unique(struct vcap_rule *rule,
2794 				    enum vcap_action_field act)
2795 {
2796 	struct vcap_rule_internal *ri = to_intrule(rule);
2797 	const struct vcap_client_actionfield *caf;
2798 
2799 	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list)
2800 		if (caf->ctrl.action == act)
2801 			return false;
2802 	return true;
2803 }
2804 
2805 /* Check if the actionfield is in the actionset */
2806 static bool vcap_actionfield_match_actionset(struct vcap_rule *rule,
2807 					     enum vcap_action_field action)
2808 {
2809 	enum vcap_actionfield_set actionset = rule->actionset;
2810 	struct vcap_rule_internal *ri = to_intrule(rule);
2811 	enum vcap_type vt = ri->admin->vtype;
2812 	const struct vcap_field *fields;
2813 
2814 	/* the field is accepted if the rule has no actionset yet */
2815 	if (actionset == VCAP_AFS_NO_VALUE)
2816 		return true;
2817 	fields = vcap_actionfields(ri->vctrl, vt, actionset);
2818 	if (!fields)
2819 		return false;
2820 	/* if there is a width there is a way */
2821 	return fields[action].width > 0;
2822 }
2823 
2824 static int vcap_rule_add_action(struct vcap_rule *rule,
2825 				enum vcap_action_field action,
2826 				enum vcap_field_type ftype,
2827 				struct vcap_client_actionfield_data *data)
2828 {
2829 	struct vcap_rule_internal *ri = to_intrule(rule);
2830 	struct vcap_client_actionfield *field;
2831 
2832 	if (!vcap_actionfield_unique(rule, action)) {
2833 		pr_warn("%s:%d: actionfield %s is already in the rule\n",
2834 			__func__, __LINE__,
2835 			vcap_actionfield_name(ri->vctrl, action));
2836 		return -EINVAL;
2837 	}
2838 
2839 	if (!vcap_actionfield_match_actionset(rule, action)) {
2840 		pr_err("%s:%d: actionfield %s does not belong in the rule actionset\n",
2841 		       __func__, __LINE__,
2842 		       vcap_actionfield_name(ri->vctrl, action));
2843 		return -EINVAL;
2844 	}
2845 
2846 	field = kzalloc(sizeof(*field), GFP_KERNEL);
2847 	if (!field)
2848 		return -ENOMEM;
2849 	memcpy(&field->data, data, sizeof(field->data));
2850 	field->ctrl.action = action;
2851 	field->ctrl.type = ftype;
2852 	list_add_tail(&field->ctrl.list, &rule->actionfields);
2853 	return 0;
2854 }
2855 
2856 static void vcap_rule_set_action_bitsize(struct vcap_u1_action *u1,
2857 					 enum vcap_bit val)
2858 {
2859 	switch (val) {
2860 	case VCAP_BIT_0:
2861 		u1->value = 0;
2862 		break;
2863 	case VCAP_BIT_1:
2864 		u1->value = 1;
2865 		break;
2866 	case VCAP_BIT_ANY:
2867 		u1->value = 0;
2868 		break;
2869 	}
2870 }
2871 
2872 /* Add a bit action with value to the rule */
2873 int vcap_rule_add_action_bit(struct vcap_rule *rule,
2874 			     enum vcap_action_field action,
2875 			     enum vcap_bit val)
2876 {
2877 	struct vcap_client_actionfield_data data;
2878 
2879 	vcap_rule_set_action_bitsize(&data.u1, val);
2880 	return vcap_rule_add_action(rule, action, VCAP_FIELD_BIT, &data);
2881 }
2882 EXPORT_SYMBOL_GPL(vcap_rule_add_action_bit);
2883 
2884 /* Add a 32 bit action field with value to the rule */
2885 int vcap_rule_add_action_u32(struct vcap_rule *rule,
2886 			     enum vcap_action_field action,
2887 			     u32 value)
2888 {
2889 	struct vcap_client_actionfield_data data;
2890 
2891 	data.u32.value = value;
2892 	return vcap_rule_add_action(rule, action, VCAP_FIELD_U32, &data);
2893 }
2894 EXPORT_SYMBOL_GPL(vcap_rule_add_action_u32);
2895 
2896 static int vcap_read_counter(struct vcap_rule_internal *ri,
2897 			     struct vcap_counter *ctr)
2898 {
2899 	struct vcap_admin *admin = ri->admin;
2900 
2901 	ri->vctrl->ops->update(ri->ndev, admin, VCAP_CMD_READ, VCAP_SEL_COUNTER,
2902 			       ri->addr);
2903 	ri->vctrl->ops->cache_read(ri->ndev, admin, VCAP_SEL_COUNTER,
2904 				   ri->counter_id, 0);
2905 	ctr->value = admin->cache.counter;
2906 	ctr->sticky = admin->cache.sticky;
2907 	return 0;
2908 }
2909 
2910 /* Copy to host byte order */
2911 void vcap_netbytes_copy(u8 *dst, u8 *src, int count)
2912 {
2913 	int idx;
2914 
2915 	for (idx = 0; idx < count; ++idx, ++dst)
2916 		*dst = src[count - idx - 1];
2917 }
2918 EXPORT_SYMBOL_GPL(vcap_netbytes_copy);
2919 
2920 /* Convert validation error code into tc extact error message */
2921 void vcap_set_tc_exterr(struct flow_cls_offload *fco, struct vcap_rule *vrule)
2922 {
2923 	switch (vrule->exterr) {
2924 	case VCAP_ERR_NONE:
2925 		break;
2926 	case VCAP_ERR_NO_ADMIN:
2927 		NL_SET_ERR_MSG_MOD(fco->common.extack,
2928 				   "Missing VCAP instance");
2929 		break;
2930 	case VCAP_ERR_NO_NETDEV:
2931 		NL_SET_ERR_MSG_MOD(fco->common.extack,
2932 				   "Missing network interface");
2933 		break;
2934 	case VCAP_ERR_NO_KEYSET_MATCH:
2935 		NL_SET_ERR_MSG_MOD(fco->common.extack,
2936 				   "No keyset matched the filter keys");
2937 		break;
2938 	case VCAP_ERR_NO_ACTIONSET_MATCH:
2939 		NL_SET_ERR_MSG_MOD(fco->common.extack,
2940 				   "No actionset matched the filter actions");
2941 		break;
2942 	case VCAP_ERR_NO_PORT_KEYSET_MATCH:
2943 		NL_SET_ERR_MSG_MOD(fco->common.extack,
2944 				   "No port keyset matched the filter keys");
2945 		break;
2946 	}
2947 }
2948 EXPORT_SYMBOL_GPL(vcap_set_tc_exterr);
2949 
2950 /* Write a rule to VCAP HW to enable it */
2951 static int vcap_enable_rule(struct vcap_rule_internal *ri)
2952 {
2953 	struct vcap_client_actionfield *af, *naf;
2954 	struct vcap_client_keyfield *kf, *nkf;
2955 	int err;
2956 
2957 	vcap_erase_cache(ri);
2958 	err = vcap_encode_rule(ri);
2959 	if (err)
2960 		goto out;
2961 	err = vcap_write_rule(ri);
2962 	if (err)
2963 		goto out;
2964 
2965 	/* Deallocate the list of keys and actions */
2966 	list_for_each_entry_safe(kf, nkf, &ri->data.keyfields, ctrl.list) {
2967 		list_del(&kf->ctrl.list);
2968 		kfree(kf);
2969 	}
2970 	list_for_each_entry_safe(af, naf, &ri->data.actionfields, ctrl.list) {
2971 		list_del(&af->ctrl.list);
2972 		kfree(af);
2973 	}
2974 	ri->state = VCAP_RS_ENABLED;
2975 out:
2976 	return err;
2977 }
2978 
2979 /* Enable all disabled rules for a specific chain/port in the VCAP HW */
2980 static int vcap_enable_rules(struct vcap_control *vctrl,
2981 			     struct net_device *ndev, int chain)
2982 {
2983 	int next_chain = chain + VCAP_CID_LOOKUP_SIZE;
2984 	struct vcap_rule_internal *ri;
2985 	struct vcap_admin *admin;
2986 	int err = 0;
2987 
2988 	list_for_each_entry(admin, &vctrl->list, list) {
2989 		if (!(chain >= admin->first_cid && chain <= admin->last_cid))
2990 			continue;
2991 
2992 		/* Found the admin, now find the offloadable rules */
2993 		mutex_lock(&admin->lock);
2994 		list_for_each_entry(ri, &admin->rules, list) {
2995 			/* Is the rule in the lookup defined by the chain */
2996 			if (!(ri->data.vcap_chain_id >= chain &&
2997 			      ri->data.vcap_chain_id < next_chain)) {
2998 				continue;
2999 			}
3000 
3001 			if (ri->ndev != ndev)
3002 				continue;
3003 
3004 			if (ri->state != VCAP_RS_DISABLED)
3005 				continue;
3006 
3007 			err = vcap_enable_rule(ri);
3008 			if (err)
3009 				break;
3010 		}
3011 		mutex_unlock(&admin->lock);
3012 		if (err)
3013 			break;
3014 	}
3015 	return err;
3016 }
3017 
3018 /* Read and erase a rule from VCAP HW to disable it */
3019 static int vcap_disable_rule(struct vcap_rule_internal *ri)
3020 {
3021 	int err;
3022 
3023 	err = vcap_read_rule(ri);
3024 	if (err)
3025 		return err;
3026 	err = vcap_decode_keyset(ri);
3027 	if (err)
3028 		return err;
3029 	err = vcap_decode_actionset(ri);
3030 	if (err)
3031 		return err;
3032 
3033 	ri->state = VCAP_RS_DISABLED;
3034 	ri->vctrl->ops->init(ri->ndev, ri->admin, ri->addr, ri->size);
3035 	return 0;
3036 }
3037 
3038 /* Disable all enabled rules for a specific chain/port in the VCAP HW */
3039 static int vcap_disable_rules(struct vcap_control *vctrl,
3040 			      struct net_device *ndev, int chain)
3041 {
3042 	struct vcap_rule_internal *ri;
3043 	struct vcap_admin *admin;
3044 	int err = 0;
3045 
3046 	list_for_each_entry(admin, &vctrl->list, list) {
3047 		if (!(chain >= admin->first_cid && chain <= admin->last_cid))
3048 			continue;
3049 
3050 		/* Found the admin, now find the rules on the chain */
3051 		mutex_lock(&admin->lock);
3052 		list_for_each_entry(ri, &admin->rules, list) {
3053 			if (ri->data.vcap_chain_id != chain)
3054 				continue;
3055 
3056 			if (ri->ndev != ndev)
3057 				continue;
3058 
3059 			if (ri->state != VCAP_RS_ENABLED)
3060 				continue;
3061 
3062 			err = vcap_disable_rule(ri);
3063 			if (err)
3064 				break;
3065 		}
3066 		mutex_unlock(&admin->lock);
3067 		if (err)
3068 			break;
3069 	}
3070 	return err;
3071 }
3072 
3073 /* Check if this port is already enabled for this VCAP instance */
3074 static bool vcap_is_enabled(struct vcap_control *vctrl, struct net_device *ndev,
3075 			    int dst_cid)
3076 {
3077 	struct vcap_enabled_port *eport;
3078 	struct vcap_admin *admin;
3079 
3080 	list_for_each_entry(admin, &vctrl->list, list)
3081 		list_for_each_entry(eport, &admin->enabled, list)
3082 			if (eport->dst_cid == dst_cid && eport->ndev == ndev)
3083 				return true;
3084 
3085 	return false;
3086 }
3087 
3088 /* Enable this port and chain id in a VCAP instance */
3089 static int vcap_enable(struct vcap_control *vctrl, struct net_device *ndev,
3090 		       unsigned long cookie, int src_cid, int dst_cid)
3091 {
3092 	struct vcap_enabled_port *eport;
3093 	struct vcap_admin *admin;
3094 
3095 	if (src_cid >= dst_cid)
3096 		return -EFAULT;
3097 
3098 	admin = vcap_find_admin(vctrl, dst_cid);
3099 	if (!admin)
3100 		return -ENOENT;
3101 
3102 	eport = kzalloc(sizeof(*eport), GFP_KERNEL);
3103 	if (!eport)
3104 		return -ENOMEM;
3105 
3106 	eport->ndev = ndev;
3107 	eport->cookie = cookie;
3108 	eport->src_cid = src_cid;
3109 	eport->dst_cid = dst_cid;
3110 	mutex_lock(&admin->lock);
3111 	list_add_tail(&eport->list, &admin->enabled);
3112 	mutex_unlock(&admin->lock);
3113 
3114 	if (vcap_path_exist(vctrl, ndev, src_cid)) {
3115 		/* Enable chained lookups */
3116 		while (dst_cid) {
3117 			admin = vcap_find_admin(vctrl, dst_cid);
3118 			if (!admin)
3119 				return -ENOENT;
3120 
3121 			vcap_enable_rules(vctrl, ndev, dst_cid);
3122 			dst_cid = vcap_get_next_chain(vctrl, ndev, dst_cid);
3123 		}
3124 	}
3125 	return 0;
3126 }
3127 
3128 /* Disable this port and chain id for a VCAP instance */
3129 static int vcap_disable(struct vcap_control *vctrl, struct net_device *ndev,
3130 			unsigned long cookie)
3131 {
3132 	struct vcap_enabled_port *elem, *eport = NULL;
3133 	struct vcap_admin *found = NULL, *admin;
3134 	int dst_cid;
3135 
3136 	list_for_each_entry(admin, &vctrl->list, list) {
3137 		list_for_each_entry(elem, &admin->enabled, list) {
3138 			if (elem->cookie == cookie && elem->ndev == ndev) {
3139 				eport = elem;
3140 				found = admin;
3141 				break;
3142 			}
3143 		}
3144 		if (eport)
3145 			break;
3146 	}
3147 
3148 	if (!eport)
3149 		return -ENOENT;
3150 
3151 	/* Disable chained lookups */
3152 	dst_cid = eport->dst_cid;
3153 	while (dst_cid) {
3154 		admin = vcap_find_admin(vctrl, dst_cid);
3155 		if (!admin)
3156 			return -ENOENT;
3157 
3158 		vcap_disable_rules(vctrl, ndev, dst_cid);
3159 		dst_cid = vcap_get_next_chain(vctrl, ndev, dst_cid);
3160 	}
3161 
3162 	mutex_lock(&found->lock);
3163 	list_del(&eport->list);
3164 	mutex_unlock(&found->lock);
3165 	kfree(eport);
3166 	return 0;
3167 }
3168 
3169 /* Enable/Disable the VCAP instance lookups */
3170 int vcap_enable_lookups(struct vcap_control *vctrl, struct net_device *ndev,
3171 			int src_cid, int dst_cid, unsigned long cookie,
3172 			bool enable)
3173 {
3174 	int err;
3175 
3176 	err = vcap_api_check(vctrl);
3177 	if (err)
3178 		return err;
3179 
3180 	if (!ndev)
3181 		return -ENODEV;
3182 
3183 	/* Source and destination must be the first chain in a lookup */
3184 	if (src_cid % VCAP_CID_LOOKUP_SIZE)
3185 		return -EFAULT;
3186 	if (dst_cid % VCAP_CID_LOOKUP_SIZE)
3187 		return -EFAULT;
3188 
3189 	if (enable) {
3190 		if (vcap_is_enabled(vctrl, ndev, dst_cid))
3191 			return -EADDRINUSE;
3192 		if (vcap_is_chain_used(vctrl, ndev, src_cid))
3193 			return -EADDRNOTAVAIL;
3194 		err = vcap_enable(vctrl, ndev, cookie, src_cid, dst_cid);
3195 	} else {
3196 		err = vcap_disable(vctrl, ndev, cookie);
3197 	}
3198 
3199 	return err;
3200 }
3201 EXPORT_SYMBOL_GPL(vcap_enable_lookups);
3202 
3203 /* Is this chain id the last lookup of all VCAPs */
3204 bool vcap_is_last_chain(struct vcap_control *vctrl, int cid, bool ingress)
3205 {
3206 	struct vcap_admin *admin;
3207 	int lookup;
3208 
3209 	if (vcap_api_check(vctrl))
3210 		return false;
3211 
3212 	admin = vcap_find_admin(vctrl, cid);
3213 	if (!admin)
3214 		return false;
3215 
3216 	if (!vcap_admin_is_last(vctrl, admin, ingress))
3217 		return false;
3218 
3219 	/* This must be the last lookup in this VCAP type */
3220 	lookup = vcap_chain_id_to_lookup(admin, cid);
3221 	return lookup == admin->lookups - 1;
3222 }
3223 EXPORT_SYMBOL_GPL(vcap_is_last_chain);
3224 
3225 /* Set a rule counter id (for certain vcaps only) */
3226 void vcap_rule_set_counter_id(struct vcap_rule *rule, u32 counter_id)
3227 {
3228 	struct vcap_rule_internal *ri = to_intrule(rule);
3229 
3230 	ri->counter_id = counter_id;
3231 }
3232 EXPORT_SYMBOL_GPL(vcap_rule_set_counter_id);
3233 
3234 int vcap_rule_set_counter(struct vcap_rule *rule, struct vcap_counter *ctr)
3235 {
3236 	struct vcap_rule_internal *ri = to_intrule(rule);
3237 	int err;
3238 
3239 	err = vcap_api_check(ri->vctrl);
3240 	if (err)
3241 		return err;
3242 	if (!ctr) {
3243 		pr_err("%s:%d: counter is missing\n", __func__, __LINE__);
3244 		return -EINVAL;
3245 	}
3246 
3247 	mutex_lock(&ri->admin->lock);
3248 	err = vcap_write_counter(ri, ctr);
3249 	mutex_unlock(&ri->admin->lock);
3250 
3251 	return err;
3252 }
3253 EXPORT_SYMBOL_GPL(vcap_rule_set_counter);
3254 
3255 int vcap_rule_get_counter(struct vcap_rule *rule, struct vcap_counter *ctr)
3256 {
3257 	struct vcap_rule_internal *ri = to_intrule(rule);
3258 	int err;
3259 
3260 	err = vcap_api_check(ri->vctrl);
3261 	if (err)
3262 		return err;
3263 	if (!ctr) {
3264 		pr_err("%s:%d: counter is missing\n", __func__, __LINE__);
3265 		return -EINVAL;
3266 	}
3267 
3268 	mutex_lock(&ri->admin->lock);
3269 	err = vcap_read_counter(ri, ctr);
3270 	mutex_unlock(&ri->admin->lock);
3271 
3272 	return err;
3273 }
3274 EXPORT_SYMBOL_GPL(vcap_rule_get_counter);
3275 
3276 /* Get a copy of a client key field */
3277 static int vcap_rule_get_key(struct vcap_rule *rule,
3278 			     enum vcap_key_field key,
3279 			     struct vcap_client_keyfield *ckf)
3280 {
3281 	struct vcap_client_keyfield *field;
3282 
3283 	field = vcap_find_keyfield(rule, key);
3284 	if (!field)
3285 		return -EINVAL;
3286 	memcpy(ckf, field, sizeof(*ckf));
3287 	INIT_LIST_HEAD(&ckf->ctrl.list);
3288 	return 0;
3289 }
3290 
3291 /* Find a keyset having the same size as the provided rule, where the keyset
3292  * does not have a type id.
3293  */
3294 static int vcap_rule_get_untyped_keyset(struct vcap_rule_internal *ri,
3295 					struct vcap_keyset_list *matches)
3296 {
3297 	struct vcap_control *vctrl = ri->vctrl;
3298 	enum vcap_type vt = ri->admin->vtype;
3299 	const struct vcap_set *keyfield_set;
3300 	int idx;
3301 
3302 	keyfield_set = vctrl->vcaps[vt].keyfield_set;
3303 	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
3304 		if (keyfield_set[idx].sw_per_item == ri->keyset_sw &&
3305 		    keyfield_set[idx].type_id == (u8)-1) {
3306 			vcap_keyset_list_add(matches, idx);
3307 			return 0;
3308 		}
3309 	}
3310 	return -EINVAL;
3311 }
3312 
3313 /* Get the keysets that matches the rule key type/mask */
3314 int vcap_rule_get_keysets(struct vcap_rule_internal *ri,
3315 			  struct vcap_keyset_list *matches)
3316 {
3317 	struct vcap_control *vctrl = ri->vctrl;
3318 	enum vcap_type vt = ri->admin->vtype;
3319 	const struct vcap_set *keyfield_set;
3320 	struct vcap_client_keyfield kf = {};
3321 	u32 value, mask;
3322 	int err, idx;
3323 
3324 	err = vcap_rule_get_key(&ri->data, VCAP_KF_TYPE, &kf);
3325 	if (err)
3326 		return vcap_rule_get_untyped_keyset(ri, matches);
3327 
3328 	if (kf.ctrl.type == VCAP_FIELD_BIT) {
3329 		value = kf.data.u1.value;
3330 		mask = kf.data.u1.mask;
3331 	} else if (kf.ctrl.type == VCAP_FIELD_U32) {
3332 		value = kf.data.u32.value;
3333 		mask = kf.data.u32.mask;
3334 	} else {
3335 		return -EINVAL;
3336 	}
3337 
3338 	keyfield_set = vctrl->vcaps[vt].keyfield_set;
3339 	for (idx = 0; idx < vctrl->vcaps[vt].keyfield_set_size; ++idx) {
3340 		if (keyfield_set[idx].sw_per_item != ri->keyset_sw)
3341 			continue;
3342 
3343 		if (keyfield_set[idx].type_id == (u8)-1) {
3344 			vcap_keyset_list_add(matches, idx);
3345 			continue;
3346 		}
3347 
3348 		if ((keyfield_set[idx].type_id & mask) == value)
3349 			vcap_keyset_list_add(matches, idx);
3350 	}
3351 	if (matches->cnt > 0)
3352 		return 0;
3353 
3354 	return -EINVAL;
3355 }
3356 
3357 /* Collect packet counts from all rules with the same cookie */
3358 int vcap_get_rule_count_by_cookie(struct vcap_control *vctrl,
3359 				  struct vcap_counter *ctr, u64 cookie)
3360 {
3361 	struct vcap_rule_internal *ri;
3362 	struct vcap_counter temp = {};
3363 	struct vcap_admin *admin;
3364 	int err;
3365 
3366 	err = vcap_api_check(vctrl);
3367 	if (err)
3368 		return err;
3369 
3370 	/* Iterate all rules in each VCAP instance */
3371 	list_for_each_entry(admin, &vctrl->list, list) {
3372 		mutex_lock(&admin->lock);
3373 		list_for_each_entry(ri, &admin->rules, list) {
3374 			if (ri->data.cookie != cookie)
3375 				continue;
3376 
3377 			err = vcap_read_counter(ri, &temp);
3378 			if (err)
3379 				goto unlock;
3380 			ctr->value += temp.value;
3381 
3382 			/* Reset the rule counter */
3383 			temp.value = 0;
3384 			temp.sticky = 0;
3385 			err = vcap_write_counter(ri, &temp);
3386 			if (err)
3387 				goto unlock;
3388 		}
3389 		mutex_unlock(&admin->lock);
3390 	}
3391 	return err;
3392 
3393 unlock:
3394 	mutex_unlock(&admin->lock);
3395 	return err;
3396 }
3397 EXPORT_SYMBOL_GPL(vcap_get_rule_count_by_cookie);
3398 
3399 static int vcap_rule_mod_key(struct vcap_rule *rule,
3400 			     enum vcap_key_field key,
3401 			     enum vcap_field_type ftype,
3402 			     struct vcap_client_keyfield_data *data)
3403 {
3404 	struct vcap_client_keyfield *field;
3405 
3406 	field = vcap_find_keyfield(rule, key);
3407 	if (!field)
3408 		return vcap_rule_add_key(rule, key, ftype, data);
3409 	memcpy(&field->data, data, sizeof(field->data));
3410 	return 0;
3411 }
3412 
3413 /* Modify a 32 bit key field with value and mask in the rule */
3414 int vcap_rule_mod_key_u32(struct vcap_rule *rule, enum vcap_key_field key,
3415 			  u32 value, u32 mask)
3416 {
3417 	struct vcap_client_keyfield_data data;
3418 
3419 	data.u32.value = value;
3420 	data.u32.mask = mask;
3421 	return vcap_rule_mod_key(rule, key, VCAP_FIELD_U32, &data);
3422 }
3423 EXPORT_SYMBOL_GPL(vcap_rule_mod_key_u32);
3424 
3425 /* Remove a key field with value and mask in the rule */
3426 int vcap_rule_rem_key(struct vcap_rule *rule, enum vcap_key_field key)
3427 {
3428 	struct vcap_rule_internal *ri = to_intrule(rule);
3429 	struct vcap_client_keyfield *field;
3430 
3431 	field = vcap_find_keyfield(rule, key);
3432 	if (!field) {
3433 		pr_err("%s:%d: key %s is not in the rule\n",
3434 		       __func__, __LINE__, vcap_keyfield_name(ri->vctrl, key));
3435 		return -EINVAL;
3436 	}
3437 	/* Deallocate the key field */
3438 	list_del(&field->ctrl.list);
3439 	kfree(field);
3440 	return 0;
3441 }
3442 EXPORT_SYMBOL_GPL(vcap_rule_rem_key);
3443 
3444 static int vcap_rule_mod_action(struct vcap_rule *rule,
3445 				enum vcap_action_field action,
3446 				enum vcap_field_type ftype,
3447 				struct vcap_client_actionfield_data *data)
3448 {
3449 	struct vcap_client_actionfield *field;
3450 
3451 	field = vcap_find_actionfield(rule, action);
3452 	if (!field)
3453 		return vcap_rule_add_action(rule, action, ftype, data);
3454 	memcpy(&field->data, data, sizeof(field->data));
3455 	return 0;
3456 }
3457 
3458 /* Modify a 32 bit action field with value in the rule */
3459 int vcap_rule_mod_action_u32(struct vcap_rule *rule,
3460 			     enum vcap_action_field action,
3461 			     u32 value)
3462 {
3463 	struct vcap_client_actionfield_data data;
3464 
3465 	data.u32.value = value;
3466 	return vcap_rule_mod_action(rule, action, VCAP_FIELD_U32, &data);
3467 }
3468 EXPORT_SYMBOL_GPL(vcap_rule_mod_action_u32);
3469 
3470 /* Drop keys in a keylist and any keys that are not supported by the keyset */
3471 int vcap_filter_rule_keys(struct vcap_rule *rule,
3472 			  enum vcap_key_field keylist[], int length,
3473 			  bool drop_unsupported)
3474 {
3475 	struct vcap_rule_internal *ri = to_intrule(rule);
3476 	struct vcap_client_keyfield *ckf, *next_ckf;
3477 	const struct vcap_field *fields;
3478 	enum vcap_key_field key;
3479 	int err = 0;
3480 	int idx;
3481 
3482 	if (length > 0) {
3483 		err = -EEXIST;
3484 		list_for_each_entry_safe(ckf, next_ckf,
3485 					 &ri->data.keyfields, ctrl.list) {
3486 			key = ckf->ctrl.key;
3487 			for (idx = 0; idx < length; ++idx)
3488 				if (key == keylist[idx]) {
3489 					list_del(&ckf->ctrl.list);
3490 					kfree(ckf);
3491 					idx++;
3492 					err = 0;
3493 				}
3494 		}
3495 	}
3496 	if (drop_unsupported) {
3497 		err = -EEXIST;
3498 		fields = vcap_keyfields(ri->vctrl, ri->admin->vtype,
3499 					rule->keyset);
3500 		if (!fields)
3501 			return err;
3502 		list_for_each_entry_safe(ckf, next_ckf,
3503 					 &ri->data.keyfields, ctrl.list) {
3504 			key = ckf->ctrl.key;
3505 			if (fields[key].width == 0) {
3506 				list_del(&ckf->ctrl.list);
3507 				kfree(ckf);
3508 				err = 0;
3509 			}
3510 		}
3511 	}
3512 	return err;
3513 }
3514 EXPORT_SYMBOL_GPL(vcap_filter_rule_keys);
3515 
3516 /* Select the keyset from the list that results in the smallest rule size */
3517 enum vcap_keyfield_set
3518 vcap_select_min_rule_keyset(struct vcap_control *vctrl,
3519 			    enum vcap_type vtype,
3520 			    struct vcap_keyset_list *kslist)
3521 {
3522 	enum vcap_keyfield_set ret = VCAP_KFS_NO_VALUE;
3523 	const struct vcap_set *kset;
3524 	int max = 100, idx;
3525 
3526 	for (idx = 0; idx < kslist->cnt; ++idx) {
3527 		kset = vcap_keyfieldset(vctrl, vtype, kslist->keysets[idx]);
3528 		if (!kset)
3529 			continue;
3530 		if (kset->sw_per_item >= max)
3531 			continue;
3532 		max = kset->sw_per_item;
3533 		ret = kslist->keysets[idx];
3534 	}
3535 	return ret;
3536 }
3537 EXPORT_SYMBOL_GPL(vcap_select_min_rule_keyset);
3538 
3539 /* Make a full copy of an existing rule with a new rule id */
3540 struct vcap_rule *vcap_copy_rule(struct vcap_rule *erule)
3541 {
3542 	struct vcap_rule_internal *ri = to_intrule(erule);
3543 	struct vcap_client_actionfield *caf;
3544 	struct vcap_client_keyfield *ckf;
3545 	struct vcap_rule *rule;
3546 	int err;
3547 
3548 	err = vcap_api_check(ri->vctrl);
3549 	if (err)
3550 		return ERR_PTR(err);
3551 
3552 	rule = vcap_alloc_rule(ri->vctrl, ri->ndev, ri->data.vcap_chain_id,
3553 			       ri->data.user, ri->data.priority, 0);
3554 	if (IS_ERR(rule))
3555 		return rule;
3556 
3557 	list_for_each_entry(ckf, &ri->data.keyfields, ctrl.list) {
3558 		/* Add a key duplicate in the new rule */
3559 		err = vcap_rule_add_key(rule,
3560 					ckf->ctrl.key,
3561 					ckf->ctrl.type,
3562 					&ckf->data);
3563 		if (err)
3564 			goto err;
3565 	}
3566 
3567 	list_for_each_entry(caf, &ri->data.actionfields, ctrl.list) {
3568 		/* Add a action duplicate in the new rule */
3569 		err = vcap_rule_add_action(rule,
3570 					   caf->ctrl.action,
3571 					   caf->ctrl.type,
3572 					   &caf->data);
3573 		if (err)
3574 			goto err;
3575 	}
3576 	return rule;
3577 err:
3578 	vcap_free_rule(rule);
3579 	return ERR_PTR(err);
3580 }
3581 EXPORT_SYMBOL_GPL(vcap_copy_rule);
3582 
3583 #ifdef CONFIG_VCAP_KUNIT_TEST
3584 #include "vcap_api_kunit.c"
3585 #endif
3586