1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Admin Function driver
3  *
4  * Copyright (C) 2020 Marvell.
5  */
6 
7 #include <linux/bitfield.h>
8 
9 #include "rvu_struct.h"
10 #include "rvu_reg.h"
11 #include "rvu.h"
12 #include "npc.h"
13 
14 #define NPC_BYTESM		GENMASK_ULL(19, 16)
15 #define NPC_HDR_OFFSET		GENMASK_ULL(15, 8)
16 #define NPC_KEY_OFFSET		GENMASK_ULL(5, 0)
17 #define NPC_LDATA_EN		BIT_ULL(7)
18 
19 static const char * const npc_flow_names[] = {
20 	[NPC_DMAC]	= "dmac",
21 	[NPC_SMAC]	= "smac",
22 	[NPC_ETYPE]	= "ether type",
23 	[NPC_VLAN_ETYPE_CTAG] = "vlan ether type ctag",
24 	[NPC_VLAN_ETYPE_STAG] = "vlan ether type stag",
25 	[NPC_OUTER_VID]	= "outer vlan id",
26 	[NPC_TOS]	= "tos",
27 	[NPC_SIP_IPV4]	= "ipv4 source ip",
28 	[NPC_DIP_IPV4]	= "ipv4 destination ip",
29 	[NPC_SIP_IPV6]	= "ipv6 source ip",
30 	[NPC_DIP_IPV6]	= "ipv6 destination ip",
31 	[NPC_IPPROTO_TCP] = "ip proto tcp",
32 	[NPC_IPPROTO_UDP] = "ip proto udp",
33 	[NPC_IPPROTO_SCTP] = "ip proto sctp",
34 	[NPC_IPPROTO_ICMP] = "ip proto icmp",
35 	[NPC_IPPROTO_ICMP6] = "ip proto icmp6",
36 	[NPC_IPPROTO_AH] = "ip proto AH",
37 	[NPC_IPPROTO_ESP] = "ip proto ESP",
38 	[NPC_SPORT_TCP]	= "tcp source port",
39 	[NPC_DPORT_TCP]	= "tcp destination port",
40 	[NPC_SPORT_UDP]	= "udp source port",
41 	[NPC_DPORT_UDP]	= "udp destination port",
42 	[NPC_SPORT_SCTP] = "sctp source port",
43 	[NPC_DPORT_SCTP] = "sctp destination port",
44 	[NPC_UNKNOWN]	= "unknown",
45 };
46 
47 const char *npc_get_field_name(u8 hdr)
48 {
49 	if (hdr >= ARRAY_SIZE(npc_flow_names))
50 		return npc_flow_names[NPC_UNKNOWN];
51 
52 	return npc_flow_names[hdr];
53 }
54 
55 /* Compute keyword masks and figure out the number of keywords a field
56  * spans in the key.
57  */
58 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type,
59 			     u8 nr_bits, int start_kwi, int offset, u8 intf)
60 {
61 	struct npc_key_field *field = &mcam->rx_key_fields[type];
62 	u8 bits_in_kw;
63 	int max_kwi;
64 
65 	if (mcam->banks_per_entry == 1)
66 		max_kwi = 1; /* NPC_MCAM_KEY_X1 */
67 	else if (mcam->banks_per_entry == 2)
68 		max_kwi = 3; /* NPC_MCAM_KEY_X2 */
69 	else
70 		max_kwi = 6; /* NPC_MCAM_KEY_X4 */
71 
72 	if (is_npc_intf_tx(intf))
73 		field = &mcam->tx_key_fields[type];
74 
75 	if (offset + nr_bits <= 64) {
76 		/* one KW only */
77 		if (start_kwi > max_kwi)
78 			return;
79 		field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0)
80 					     << offset;
81 		field->nr_kws = 1;
82 	} else if (offset + nr_bits > 64 &&
83 		   offset + nr_bits <= 128) {
84 		/* two KWs */
85 		if (start_kwi + 1 > max_kwi)
86 			return;
87 		/* first KW mask */
88 		bits_in_kw = 64 - offset;
89 		field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
90 					     << offset;
91 		/* second KW mask i.e. mask for rest of bits */
92 		bits_in_kw = nr_bits + offset - 64;
93 		field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0);
94 		field->nr_kws = 2;
95 	} else {
96 		/* three KWs */
97 		if (start_kwi + 2 > max_kwi)
98 			return;
99 		/* first KW mask */
100 		bits_in_kw = 64 - offset;
101 		field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0)
102 					     << offset;
103 		/* second KW mask */
104 		field->kw_mask[start_kwi + 1] = ~0ULL;
105 		/* third KW mask i.e. mask for rest of bits */
106 		bits_in_kw = nr_bits + offset - 128;
107 		field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0);
108 		field->nr_kws = 3;
109 	}
110 }
111 
112 /* Helper function to figure out whether field exists in the key */
113 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf)
114 {
115 	struct npc_mcam *mcam = &rvu->hw->mcam;
116 	struct npc_key_field *input;
117 
118 	input  = &mcam->rx_key_fields[type];
119 	if (is_npc_intf_tx(intf))
120 		input  = &mcam->tx_key_fields[type];
121 
122 	return input->nr_kws > 0;
123 }
124 
125 static bool npc_is_same(struct npc_key_field *input,
126 			struct npc_key_field *field)
127 {
128 	return memcmp(&input->layer_mdata, &field->layer_mdata,
129 		     sizeof(struct npc_layer_mdata)) == 0;
130 }
131 
132 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type,
133 				u64 cfg, u8 lid, u8 lt, u8 intf)
134 {
135 	struct npc_key_field *input = &mcam->rx_key_fields[type];
136 
137 	if (is_npc_intf_tx(intf))
138 		input = &mcam->tx_key_fields[type];
139 
140 	input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
141 	input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg);
142 	input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1;
143 	input->layer_mdata.ltype = lt;
144 	input->layer_mdata.lid = lid;
145 }
146 
147 static bool npc_check_overlap_fields(struct npc_key_field *input1,
148 				     struct npc_key_field *input2)
149 {
150 	int kwi;
151 
152 	/* Fields with same layer id and different ltypes are mutually
153 	 * exclusive hence they can be overlapped
154 	 */
155 	if (input1->layer_mdata.lid == input2->layer_mdata.lid &&
156 	    input1->layer_mdata.ltype != input2->layer_mdata.ltype)
157 		return false;
158 
159 	for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) {
160 		if (input1->kw_mask[kwi] & input2->kw_mask[kwi])
161 			return true;
162 	}
163 
164 	return false;
165 }
166 
167 /* Helper function to check whether given field overlaps with any other fields
168  * in the key. Due to limitations on key size and the key extraction profile in
169  * use higher layers can overwrite lower layer's header fields. Hence overlap
170  * needs to be checked.
171  */
172 static bool npc_check_overlap(struct rvu *rvu, int blkaddr,
173 			      enum key_fields type, u8 start_lid, u8 intf)
174 {
175 	struct npc_mcam *mcam = &rvu->hw->mcam;
176 	struct npc_key_field *dummy, *input;
177 	int start_kwi, offset;
178 	u8 nr_bits, lid, lt, ld;
179 	u64 cfg;
180 
181 	dummy = &mcam->rx_key_fields[NPC_UNKNOWN];
182 	input = &mcam->rx_key_fields[type];
183 
184 	if (is_npc_intf_tx(intf)) {
185 		dummy = &mcam->tx_key_fields[NPC_UNKNOWN];
186 		input = &mcam->tx_key_fields[type];
187 	}
188 
189 	for (lid = start_lid; lid < NPC_MAX_LID; lid++) {
190 		for (lt = 0; lt < NPC_MAX_LT; lt++) {
191 			for (ld = 0; ld < NPC_MAX_LD; ld++) {
192 				cfg = rvu_read64(rvu, blkaddr,
193 						 NPC_AF_INTFX_LIDX_LTX_LDX_CFG
194 						 (intf, lid, lt, ld));
195 				if (!FIELD_GET(NPC_LDATA_EN, cfg))
196 					continue;
197 				memset(dummy, 0, sizeof(struct npc_key_field));
198 				npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg,
199 						    lid, lt, intf);
200 				/* exclude input */
201 				if (npc_is_same(input, dummy))
202 					continue;
203 				start_kwi = dummy->layer_mdata.key / 8;
204 				offset = (dummy->layer_mdata.key * 8) % 64;
205 				nr_bits = dummy->layer_mdata.len * 8;
206 				/* form KW masks */
207 				npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits,
208 						 start_kwi, offset, intf);
209 				/* check any input field bits falls in any
210 				 * other field bits.
211 				 */
212 				if (npc_check_overlap_fields(dummy, input))
213 					return true;
214 			}
215 		}
216 	}
217 
218 	return false;
219 }
220 
221 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type,
222 			    u8 intf)
223 {
224 	if (!npc_is_field_present(rvu, type, intf) ||
225 	    npc_check_overlap(rvu, blkaddr, type, 0, intf))
226 		return false;
227 	return true;
228 }
229 
230 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number,
231 				  u8 key_nibble, u8 intf)
232 {
233 	u8 offset = (key_nibble * 4) % 64; /* offset within key word */
234 	u8 kwi = (key_nibble * 4) / 64; /* which word in key */
235 	u8 nr_bits = 4; /* bits in a nibble */
236 	u8 type;
237 
238 	switch (bit_number) {
239 	case 0 ... 2:
240 		type = NPC_CHAN;
241 		break;
242 	case 3:
243 		type = NPC_ERRLEV;
244 		break;
245 	case 4 ... 5:
246 		type = NPC_ERRCODE;
247 		break;
248 	case 6:
249 		type = NPC_LXMB;
250 		break;
251 	/* check for LTYPE only as of now */
252 	case 9:
253 		type = NPC_LA;
254 		break;
255 	case 12:
256 		type = NPC_LB;
257 		break;
258 	case 15:
259 		type = NPC_LC;
260 		break;
261 	case 18:
262 		type = NPC_LD;
263 		break;
264 	case 21:
265 		type = NPC_LE;
266 		break;
267 	case 24:
268 		type = NPC_LF;
269 		break;
270 	case 27:
271 		type = NPC_LG;
272 		break;
273 	case 30:
274 		type = NPC_LH;
275 		break;
276 	default:
277 		return;
278 	}
279 	npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf);
280 }
281 
282 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf)
283 {
284 	struct npc_mcam *mcam = &rvu->hw->mcam;
285 	struct npc_key_field *key_fields;
286 	/* Ether type can come from three layers
287 	 * (ethernet, single tagged, double tagged)
288 	 */
289 	struct npc_key_field *etype_ether;
290 	struct npc_key_field *etype_tag1;
291 	struct npc_key_field *etype_tag2;
292 	/* Outer VLAN TCI can come from two layers
293 	 * (single tagged, double tagged)
294 	 */
295 	struct npc_key_field *vlan_tag1;
296 	struct npc_key_field *vlan_tag2;
297 	u64 *features;
298 	u8 start_lid;
299 	int i;
300 
301 	key_fields = mcam->rx_key_fields;
302 	features = &mcam->rx_features;
303 
304 	if (is_npc_intf_tx(intf)) {
305 		key_fields = mcam->tx_key_fields;
306 		features = &mcam->tx_features;
307 	}
308 
309 	/* Handle header fields which can come from multiple layers like
310 	 * etype, outer vlan tci. These fields should have same position in
311 	 * the key otherwise to install a mcam rule more than one entry is
312 	 * needed which complicates mcam space management.
313 	 */
314 	etype_ether = &key_fields[NPC_ETYPE_ETHER];
315 	etype_tag1 = &key_fields[NPC_ETYPE_TAG1];
316 	etype_tag2 = &key_fields[NPC_ETYPE_TAG2];
317 	vlan_tag1 = &key_fields[NPC_VLAN_TAG1];
318 	vlan_tag2 = &key_fields[NPC_VLAN_TAG2];
319 
320 	/* if key profile programmed does not extract Ethertype at all */
321 	if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
322 		goto vlan_tci;
323 
324 	/* if key profile programmed extracts Ethertype from one layer */
325 	if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws)
326 		key_fields[NPC_ETYPE] = *etype_ether;
327 	if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws)
328 		key_fields[NPC_ETYPE] = *etype_tag1;
329 	if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws)
330 		key_fields[NPC_ETYPE] = *etype_tag2;
331 
332 	/* if key profile programmed extracts Ethertype from multiple layers */
333 	if (etype_ether->nr_kws && etype_tag1->nr_kws) {
334 		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
335 			if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i])
336 				goto vlan_tci;
337 		}
338 		key_fields[NPC_ETYPE] = *etype_tag1;
339 	}
340 	if (etype_ether->nr_kws && etype_tag2->nr_kws) {
341 		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
342 			if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i])
343 				goto vlan_tci;
344 		}
345 		key_fields[NPC_ETYPE] = *etype_tag2;
346 	}
347 	if (etype_tag1->nr_kws && etype_tag2->nr_kws) {
348 		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
349 			if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i])
350 				goto vlan_tci;
351 		}
352 		key_fields[NPC_ETYPE] = *etype_tag2;
353 	}
354 
355 	/* check none of higher layers overwrite Ethertype */
356 	start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1;
357 	if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf))
358 		goto vlan_tci;
359 	*features |= BIT_ULL(NPC_ETYPE);
360 vlan_tci:
361 	/* if key profile does not extract outer vlan tci at all */
362 	if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
363 		goto done;
364 
365 	/* if key profile extracts outer vlan tci from one layer */
366 	if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws)
367 		key_fields[NPC_OUTER_VID] = *vlan_tag1;
368 	if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws)
369 		key_fields[NPC_OUTER_VID] = *vlan_tag2;
370 
371 	/* if key profile extracts outer vlan tci from multiple layers */
372 	if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) {
373 		for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
374 			if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i])
375 				goto done;
376 		}
377 		key_fields[NPC_OUTER_VID] = *vlan_tag2;
378 	}
379 	/* check none of higher layers overwrite outer vlan tci */
380 	start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1;
381 	if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf))
382 		goto done;
383 	*features |= BIT_ULL(NPC_OUTER_VID);
384 done:
385 	return;
386 }
387 
388 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid,
389 			   u8 lt, u64 cfg, u8 intf)
390 {
391 	struct npc_mcam *mcam = &rvu->hw->mcam;
392 	u8 hdr, key, nr_bytes, bit_offset;
393 	u8 la_ltype, la_start;
394 	/* starting KW index and starting bit position */
395 	int start_kwi, offset;
396 
397 	nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1;
398 	hdr = FIELD_GET(NPC_HDR_OFFSET, cfg);
399 	key = FIELD_GET(NPC_KEY_OFFSET, cfg);
400 	start_kwi = key / 8;
401 	offset = (key * 8) % 64;
402 
403 	/* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding
404 	 * ethernet header.
405 	 */
406 	if (is_npc_intf_tx(intf)) {
407 		la_ltype = NPC_LT_LA_IH_NIX_ETHER;
408 		la_start = 8;
409 	} else {
410 		la_ltype = NPC_LT_LA_ETHER;
411 		la_start = 0;
412 	}
413 
414 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen)			       \
415 do {									       \
416 	if (lid == (hlid) && lt == (hlt)) {				       \
417 		if ((hstart) >= hdr &&					       \
418 		    ((hstart) + (hlen)) <= (hdr + nr_bytes)) {	               \
419 			bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \
420 			npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \
421 			npc_set_kw_masks(mcam, (name), (hlen) * 8,	       \
422 					 start_kwi, offset + bit_offset, intf);\
423 		}							       \
424 	}								       \
425 } while (0)
426 
427 	/* List LID, LTYPE, start offset from layer and length(in bytes) of
428 	 * packet header fields below.
429 	 * Example: Source IP is 4 bytes and starts at 12th byte of IP header
430 	 */
431 	NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1);
432 	NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4);
433 	NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4);
434 	NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16);
435 	NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16);
436 	NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2);
437 	NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2);
438 	NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2);
439 	NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2);
440 	NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2);
441 	NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2);
442 	NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2);
443 	NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2);
444 	NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2);
445 	NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2);
446 	NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2);
447 	NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6);
448 	NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6);
449 	/* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */
450 	NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2);
451 }
452 
453 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf)
454 {
455 	struct npc_mcam *mcam = &rvu->hw->mcam;
456 	u64 *features = &mcam->rx_features;
457 	u64 tcp_udp_sctp;
458 	int hdr;
459 
460 	if (is_npc_intf_tx(intf))
461 		features = &mcam->tx_features;
462 
463 	for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) {
464 		if (npc_check_field(rvu, blkaddr, hdr, intf))
465 			*features |= BIT_ULL(hdr);
466 	}
467 
468 	tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) |
469 		       BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) |
470 		       BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP);
471 
472 	/* for tcp/udp/sctp corresponding layer type should be in the key */
473 	if (*features & tcp_udp_sctp) {
474 		if (!npc_check_field(rvu, blkaddr, NPC_LD, intf))
475 			*features &= ~tcp_udp_sctp;
476 		else
477 			*features |= BIT_ULL(NPC_IPPROTO_TCP) |
478 				     BIT_ULL(NPC_IPPROTO_UDP) |
479 				     BIT_ULL(NPC_IPPROTO_SCTP);
480 	}
481 
482 	/* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */
483 	if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) {
484 		*features |= BIT_ULL(NPC_IPPROTO_AH);
485 		*features |= BIT_ULL(NPC_IPPROTO_ICMP);
486 		*features |= BIT_ULL(NPC_IPPROTO_ICMP6);
487 	}
488 
489 	/* for ESP, check if corresponding layer type is present in the key */
490 	if (npc_check_field(rvu, blkaddr, NPC_LE, intf))
491 		*features |= BIT_ULL(NPC_IPPROTO_ESP);
492 
493 	/* for vlan corresponding layer type should be in the key */
494 	if (*features & BIT_ULL(NPC_OUTER_VID))
495 		if (!npc_check_field(rvu, blkaddr, NPC_LB, intf))
496 			*features &= ~BIT_ULL(NPC_OUTER_VID);
497 
498 	/* for vlan ethertypes corresponding layer type should be in the key */
499 	if (npc_check_field(rvu, blkaddr, NPC_LB, intf))
500 		*features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) |
501 			     BIT_ULL(NPC_VLAN_ETYPE_STAG);
502 }
503 
504 /* Scan key extraction profile and record how fields of our interest
505  * fill the key structure. Also verify Channel and DMAC exists in
506  * key and not overwritten by other header fields.
507  */
508 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf)
509 {
510 	struct npc_mcam *mcam = &rvu->hw->mcam;
511 	u8 lid, lt, ld, bitnr;
512 	u8 key_nibble = 0;
513 	u64 cfg;
514 
515 	/* Scan and note how parse result is going to be in key.
516 	 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from
517 	 * parse result in the key. The enabled nibbles from parse result
518 	 * will be concatenated in key.
519 	 */
520 	cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
521 	cfg &= NPC_PARSE_NIBBLE;
522 	for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) {
523 		npc_scan_parse_result(mcam, bitnr, key_nibble, intf);
524 		key_nibble++;
525 	}
526 
527 	/* Scan and note how layer data is going to be in key */
528 	for (lid = 0; lid < NPC_MAX_LID; lid++) {
529 		for (lt = 0; lt < NPC_MAX_LT; lt++) {
530 			for (ld = 0; ld < NPC_MAX_LD; ld++) {
531 				cfg = rvu_read64(rvu, blkaddr,
532 						 NPC_AF_INTFX_LIDX_LTX_LDX_CFG
533 						 (intf, lid, lt, ld));
534 				if (!FIELD_GET(NPC_LDATA_EN, cfg))
535 					continue;
536 				npc_scan_ldata(rvu, blkaddr, lid, lt, cfg,
537 					       intf);
538 			}
539 		}
540 	}
541 
542 	return 0;
543 }
544 
545 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr)
546 {
547 	int err;
548 
549 	err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX);
550 	if (err)
551 		return err;
552 
553 	err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX);
554 	if (err)
555 		return err;
556 
557 	/* Channel is mandatory */
558 	if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) {
559 		dev_err(rvu->dev, "Channel not present in Key\n");
560 		return -EINVAL;
561 	}
562 	/* check that none of the fields overwrite channel */
563 	if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) {
564 		dev_err(rvu->dev, "Channel cannot be overwritten\n");
565 		return -EINVAL;
566 	}
567 	/* DMAC should be present in key for unicast filter to work */
568 	if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) {
569 		dev_err(rvu->dev, "DMAC not present in Key\n");
570 		return -EINVAL;
571 	}
572 	/* check that none of the fields overwrite DMAC */
573 	if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) {
574 		dev_err(rvu->dev, "DMAC cannot be overwritten\n");
575 		return -EINVAL;
576 	}
577 
578 	npc_set_features(rvu, blkaddr, NIX_INTF_TX);
579 	npc_set_features(rvu, blkaddr, NIX_INTF_RX);
580 	npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX);
581 	npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX);
582 
583 	return 0;
584 }
585 
586 int npc_flow_steering_init(struct rvu *rvu, int blkaddr)
587 {
588 	struct npc_mcam *mcam = &rvu->hw->mcam;
589 
590 	INIT_LIST_HEAD(&mcam->mcam_rules);
591 
592 	return npc_scan_verify_kex(rvu, blkaddr);
593 }
594 
595 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf)
596 {
597 	struct npc_mcam *mcam = &rvu->hw->mcam;
598 	u64 *mcam_features = &mcam->rx_features;
599 	u64 unsupported;
600 	u8 bit;
601 
602 	if (is_npc_intf_tx(intf))
603 		mcam_features = &mcam->tx_features;
604 
605 	unsupported = (*mcam_features ^ features) & ~(*mcam_features);
606 	if (unsupported) {
607 		dev_info(rvu->dev, "Unsupported flow(s):\n");
608 		for_each_set_bit(bit, (unsigned long *)&unsupported, 64)
609 			dev_info(rvu->dev, "%s ", npc_get_field_name(bit));
610 		return -EOPNOTSUPP;
611 	}
612 
613 	return 0;
614 }
615 
616 /* npc_update_entry - Based on the masks generated during
617  * the key scanning, updates the given entry with value and
618  * masks for the field of interest. Maximum 16 bytes of a packet
619  * header can be extracted by HW hence lo and hi are sufficient.
620  * When field bytes are less than or equal to 8 then hi should be
621  * 0 for value and mask.
622  *
623  * If exact match of value is required then mask should be all 1's.
624  * If any bits in mask are 0 then corresponding bits in value are
625  * dont care.
626  */
627 static void npc_update_entry(struct rvu *rvu, enum key_fields type,
628 			     struct mcam_entry *entry, u64 val_lo,
629 			     u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf)
630 {
631 	struct npc_mcam *mcam = &rvu->hw->mcam;
632 	struct mcam_entry dummy = { {0} };
633 	struct npc_key_field *field;
634 	u64 kw1, kw2, kw3;
635 	u8 shift;
636 	int i;
637 
638 	field = &mcam->rx_key_fields[type];
639 	if (is_npc_intf_tx(intf))
640 		field = &mcam->tx_key_fields[type];
641 
642 	if (!field->nr_kws)
643 		return;
644 
645 	for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
646 		if (!field->kw_mask[i])
647 			continue;
648 		/* place key value in kw[x] */
649 		shift = __ffs64(field->kw_mask[i]);
650 		/* update entry value */
651 		kw1 = (val_lo << shift) & field->kw_mask[i];
652 		dummy.kw[i] = kw1;
653 		/* update entry mask */
654 		kw1 = (mask_lo << shift) & field->kw_mask[i];
655 		dummy.kw_mask[i] = kw1;
656 
657 		if (field->nr_kws == 1)
658 			break;
659 		/* place remaining bits of key value in kw[x + 1] */
660 		if (field->nr_kws == 2) {
661 			/* update entry value */
662 			kw2 = shift ? val_lo >> (64 - shift) : 0;
663 			kw2 |= (val_hi << shift);
664 			kw2 &= field->kw_mask[i + 1];
665 			dummy.kw[i + 1] = kw2;
666 			/* update entry mask */
667 			kw2 = shift ? mask_lo >> (64 - shift) : 0;
668 			kw2 |= (mask_hi << shift);
669 			kw2 &= field->kw_mask[i + 1];
670 			dummy.kw_mask[i + 1] = kw2;
671 			break;
672 		}
673 		/* place remaining bits of key value in kw[x + 1], kw[x + 2] */
674 		if (field->nr_kws == 3) {
675 			/* update entry value */
676 			kw2 = shift ? val_lo >> (64 - shift) : 0;
677 			kw2 |= (val_hi << shift);
678 			kw2 &= field->kw_mask[i + 1];
679 			kw3 = shift ? val_hi >> (64 - shift) : 0;
680 			kw3 &= field->kw_mask[i + 2];
681 			dummy.kw[i + 1] = kw2;
682 			dummy.kw[i + 2] = kw3;
683 			/* update entry mask */
684 			kw2 = shift ? mask_lo >> (64 - shift) : 0;
685 			kw2 |= (mask_hi << shift);
686 			kw2 &= field->kw_mask[i + 1];
687 			kw3 = shift ? mask_hi >> (64 - shift) : 0;
688 			kw3 &= field->kw_mask[i + 2];
689 			dummy.kw_mask[i + 1] = kw2;
690 			dummy.kw_mask[i + 2] = kw3;
691 			break;
692 		}
693 	}
694 	/* dummy is ready with values and masks for given key
695 	 * field now clear and update input entry with those
696 	 */
697 	for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) {
698 		if (!field->kw_mask[i])
699 			continue;
700 		entry->kw[i] &= ~field->kw_mask[i];
701 		entry->kw_mask[i] &= ~field->kw_mask[i];
702 
703 		entry->kw[i] |= dummy.kw[i];
704 		entry->kw_mask[i] |= dummy.kw_mask[i];
705 	}
706 }
707 
708 #define IPV6_WORDS     4
709 
710 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry,
711 				 u64 features, struct flow_msg *pkt,
712 				 struct flow_msg *mask,
713 				 struct rvu_npc_mcam_rule *output, u8 intf)
714 {
715 	u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS];
716 	u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS];
717 	struct flow_msg *opkt = &output->packet;
718 	struct flow_msg *omask = &output->mask;
719 	u64 mask_lo, mask_hi;
720 	u64 val_lo, val_hi;
721 
722 	/* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet
723 	 * values to be programmed in MCAM should as below:
724 	 * val_high: 0xfe80000000000000
725 	 * val_low: 0x2c6863fffe5e2d0a
726 	 */
727 	if (features & BIT_ULL(NPC_SIP_IPV6)) {
728 		be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS);
729 		be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS);
730 
731 		mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1];
732 		mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3];
733 		val_hi = (u64)src_ip[0] << 32 | src_ip[1];
734 		val_lo = (u64)src_ip[2] << 32 | src_ip[3];
735 
736 		npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi,
737 				 mask_lo, mask_hi, intf);
738 		memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src));
739 		memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src));
740 	}
741 	if (features & BIT_ULL(NPC_DIP_IPV6)) {
742 		be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS);
743 		be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS);
744 
745 		mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1];
746 		mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3];
747 		val_hi = (u64)dst_ip[0] << 32 | dst_ip[1];
748 		val_lo = (u64)dst_ip[2] << 32 | dst_ip[3];
749 
750 		npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi,
751 				 mask_lo, mask_hi, intf);
752 		memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst));
753 		memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst));
754 	}
755 }
756 
757 static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry,
758 				     u64 features, u8 intf)
759 {
760 	bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG));
761 	bool stag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_STAG));
762 	bool vid = !!(features & BIT_ULL(NPC_OUTER_VID));
763 
764 	/* If only VLAN id is given then always match outer VLAN id */
765 	if (vid && !ctag && !stag) {
766 		npc_update_entry(rvu, NPC_LB, entry,
767 				 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0,
768 				 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf);
769 		return;
770 	}
771 	if (ctag)
772 		npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0,
773 				 ~0ULL, 0, intf);
774 	if (stag)
775 		npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0,
776 				 ~0ULL, 0, intf);
777 }
778 
779 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry,
780 			    u64 features, struct flow_msg *pkt,
781 			    struct flow_msg *mask,
782 			    struct rvu_npc_mcam_rule *output, u8 intf)
783 {
784 	u64 dmac_mask = ether_addr_to_u64(mask->dmac);
785 	u64 smac_mask = ether_addr_to_u64(mask->smac);
786 	u64 dmac_val = ether_addr_to_u64(pkt->dmac);
787 	u64 smac_val = ether_addr_to_u64(pkt->smac);
788 	struct flow_msg *opkt = &output->packet;
789 	struct flow_msg *omask = &output->mask;
790 
791 	if (!features)
792 		return;
793 
794 	/* For tcp/udp/sctp LTYPE should be present in entry */
795 	if (features & BIT_ULL(NPC_IPPROTO_TCP))
796 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP,
797 				 0, ~0ULL, 0, intf);
798 	if (features & BIT_ULL(NPC_IPPROTO_UDP))
799 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP,
800 				 0, ~0ULL, 0, intf);
801 	if (features & BIT_ULL(NPC_IPPROTO_SCTP))
802 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP,
803 				 0, ~0ULL, 0, intf);
804 	if (features & BIT_ULL(NPC_IPPROTO_ICMP))
805 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP,
806 				 0, ~0ULL, 0, intf);
807 	if (features & BIT_ULL(NPC_IPPROTO_ICMP6))
808 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6,
809 				 0, ~0ULL, 0, intf);
810 
811 	/* For AH, LTYPE should be present in entry */
812 	if (features & BIT_ULL(NPC_IPPROTO_AH))
813 		npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH,
814 				 0, ~0ULL, 0, intf);
815 	/* For ESP, LTYPE should be present in entry */
816 	if (features & BIT_ULL(NPC_IPPROTO_ESP))
817 		npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP,
818 				 0, ~0ULL, 0, intf);
819 
820 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi)	      \
821 do {									      \
822 	if (features & BIT_ULL((field))) {				      \
823 		npc_update_entry(rvu, (field), entry, (val_lo), (val_hi),     \
824 				 (mask_lo), (mask_hi), intf);		      \
825 		memcpy(&opkt->member, &pkt->member, sizeof(pkt->member));     \
826 		memcpy(&omask->member, &mask->member, sizeof(mask->member));  \
827 	}								      \
828 } while (0)
829 
830 	NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0);
831 	NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0);
832 	NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0,
833 		       ntohs(mask->etype), 0);
834 	NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0);
835 	NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0,
836 		       ntohl(mask->ip4src), 0);
837 	NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0,
838 		       ntohl(mask->ip4dst), 0);
839 	NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0,
840 		       ntohs(mask->sport), 0);
841 	NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0,
842 		       ntohs(mask->sport), 0);
843 	NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0,
844 		       ntohs(mask->dport), 0);
845 	NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0,
846 		       ntohs(mask->dport), 0);
847 	NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0,
848 		       ntohs(mask->sport), 0);
849 	NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0,
850 		       ntohs(mask->dport), 0);
851 
852 	NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0,
853 		       ntohs(mask->vlan_tci), 0);
854 
855 	npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf);
856 	npc_update_vlan_features(rvu, entry, features, intf);
857 }
858 
859 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam,
860 						    u16 entry)
861 {
862 	struct rvu_npc_mcam_rule *iter;
863 
864 	mutex_lock(&mcam->lock);
865 	list_for_each_entry(iter, &mcam->mcam_rules, list) {
866 		if (iter->entry == entry) {
867 			mutex_unlock(&mcam->lock);
868 			return iter;
869 		}
870 	}
871 	mutex_unlock(&mcam->lock);
872 
873 	return NULL;
874 }
875 
876 static void rvu_mcam_add_rule(struct npc_mcam *mcam,
877 			      struct rvu_npc_mcam_rule *rule)
878 {
879 	struct list_head *head = &mcam->mcam_rules;
880 	struct rvu_npc_mcam_rule *iter;
881 
882 	mutex_lock(&mcam->lock);
883 	list_for_each_entry(iter, &mcam->mcam_rules, list) {
884 		if (iter->entry > rule->entry)
885 			break;
886 		head = &iter->list;
887 	}
888 
889 	list_add(&rule->list, head);
890 	mutex_unlock(&mcam->lock);
891 }
892 
893 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc,
894 					      struct rvu_npc_mcam_rule *rule)
895 {
896 	struct npc_mcam_oper_counter_req free_req = { 0 };
897 	struct msg_rsp free_rsp;
898 
899 	if (!rule->has_cntr)
900 		return;
901 
902 	free_req.hdr.pcifunc = pcifunc;
903 	free_req.cntr = rule->cntr;
904 
905 	rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp);
906 	rule->has_cntr = false;
907 }
908 
909 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc,
910 					 struct rvu_npc_mcam_rule *rule,
911 					 struct npc_install_flow_rsp *rsp)
912 {
913 	struct npc_mcam_alloc_counter_req cntr_req = { 0 };
914 	struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 };
915 	int err;
916 
917 	cntr_req.hdr.pcifunc = pcifunc;
918 	cntr_req.contig = true;
919 	cntr_req.count = 1;
920 
921 	/* we try to allocate a counter to track the stats of this
922 	 * rule. If counter could not be allocated then proceed
923 	 * without counter because counters are limited than entries.
924 	 */
925 	err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req,
926 						      &cntr_rsp);
927 	if (!err && cntr_rsp.count) {
928 		rule->cntr = cntr_rsp.cntr;
929 		rule->has_cntr = true;
930 		rsp->counter = rule->cntr;
931 	} else {
932 		rsp->counter = err;
933 	}
934 }
935 
936 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
937 				struct mcam_entry *entry,
938 				struct npc_install_flow_req *req,
939 				u16 target, bool pf_set_vfs_mac)
940 {
941 	struct rvu_switch *rswitch = &rvu->rswitch;
942 	struct nix_rx_action action;
943 
944 	if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac)
945 		req->chan_mask = 0x0; /* Do not care channel */
946 
947 	npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask,
948 			 0, NIX_INTF_RX);
949 
950 	*(u64 *)&action = 0x00;
951 	action.pf_func = target;
952 	action.op = req->op;
953 	action.index = req->index;
954 	action.match_id = req->match_id;
955 	action.flow_key_alg = req->flow_key_alg;
956 
957 	if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule)
958 		action = pfvf->def_ucast_rule->rx_action;
959 
960 	entry->action = *(u64 *)&action;
961 
962 	/* VTAG0 starts at 0th byte of LID_B.
963 	 * VTAG1 starts at 4th byte of LID_B.
964 	 */
965 	entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) |
966 			     FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) |
967 			     FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) |
968 			     FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) |
969 			     FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) |
970 			     FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) |
971 			     FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) |
972 			     FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4);
973 }
974 
975 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf,
976 				struct mcam_entry *entry,
977 				struct npc_install_flow_req *req, u16 target)
978 {
979 	struct nix_tx_action action;
980 	u64 mask = ~0ULL;
981 
982 	/* If AF is installing then do not care about
983 	 * PF_FUNC in Send Descriptor
984 	 */
985 	if (is_pffunc_af(req->hdr.pcifunc))
986 		mask = 0;
987 
988 	npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target),
989 			 0, mask, 0, NIX_INTF_TX);
990 
991 	*(u64 *)&action = 0x00;
992 	action.op = req->op;
993 	action.index = req->index;
994 	action.match_id = req->match_id;
995 
996 	entry->action = *(u64 *)&action;
997 
998 	/* VTAG0 starts at 0th byte of LID_B.
999 	 * VTAG1 starts at 4th byte of LID_B.
1000 	 */
1001 	entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) |
1002 			     FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) |
1003 			     FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) |
1004 			     FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) |
1005 			     FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) |
1006 			     FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) |
1007 			     FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) |
1008 			     FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24);
1009 }
1010 
1011 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target,
1012 			    int nixlf, struct rvu_pfvf *pfvf,
1013 			    struct npc_install_flow_req *req,
1014 			    struct npc_install_flow_rsp *rsp, bool enable,
1015 			    bool pf_set_vfs_mac)
1016 {
1017 	struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule;
1018 	u64 features, installed_features, missing_features = 0;
1019 	struct npc_mcam_write_entry_req write_req = { 0 };
1020 	struct npc_mcam *mcam = &rvu->hw->mcam;
1021 	struct rvu_npc_mcam_rule dummy = { 0 };
1022 	struct rvu_npc_mcam_rule *rule;
1023 	u16 owner = req->hdr.pcifunc;
1024 	struct msg_rsp write_rsp;
1025 	struct mcam_entry *entry;
1026 	int entry_index, err;
1027 	bool new = false;
1028 
1029 	installed_features = req->features;
1030 	features = req->features;
1031 	entry = &write_req.entry_data;
1032 	entry_index = req->entry;
1033 
1034 	npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy,
1035 			req->intf);
1036 
1037 	if (is_npc_intf_rx(req->intf))
1038 		npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac);
1039 	else
1040 		npc_update_tx_entry(rvu, pfvf, entry, req, target);
1041 
1042 	/* Default unicast rules do not exist for TX */
1043 	if (is_npc_intf_tx(req->intf))
1044 		goto find_rule;
1045 
1046 	if (req->default_rule) {
1047 		entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf,
1048 						       NIXLF_UCAST_ENTRY);
1049 		enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index);
1050 	}
1051 
1052 	/* update mcam entry with default unicast rule attributes */
1053 	if (def_ucast_rule && (req->default_rule && req->append)) {
1054 		missing_features = (def_ucast_rule->features ^ features) &
1055 					def_ucast_rule->features;
1056 		if (missing_features)
1057 			npc_update_flow(rvu, entry, missing_features,
1058 					&def_ucast_rule->packet,
1059 					&def_ucast_rule->mask,
1060 					&dummy, req->intf);
1061 		installed_features = req->features | missing_features;
1062 	}
1063 
1064 find_rule:
1065 	rule = rvu_mcam_find_rule(mcam, entry_index);
1066 	if (!rule) {
1067 		rule = kzalloc(sizeof(*rule), GFP_KERNEL);
1068 		if (!rule)
1069 			return -ENOMEM;
1070 		new = true;
1071 	}
1072 
1073 	/* allocate new counter if rule has no counter */
1074 	if (!req->default_rule && req->set_cntr && !rule->has_cntr)
1075 		rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp);
1076 
1077 	/* if user wants to delete an existing counter for a rule then
1078 	 * free the counter
1079 	 */
1080 	if (!req->set_cntr && rule->has_cntr)
1081 		rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1082 
1083 	write_req.hdr.pcifunc = owner;
1084 
1085 	/* AF owns the default rules so change the owner just to relax
1086 	 * the checks in rvu_mbox_handler_npc_mcam_write_entry
1087 	 */
1088 	if (req->default_rule)
1089 		write_req.hdr.pcifunc = 0;
1090 
1091 	write_req.entry = entry_index;
1092 	write_req.intf = req->intf;
1093 	write_req.enable_entry = (u8)enable;
1094 	/* if counter is available then clear and use it */
1095 	if (req->set_cntr && rule->has_cntr) {
1096 		rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00);
1097 		write_req.set_cntr = 1;
1098 		write_req.cntr = rule->cntr;
1099 	}
1100 
1101 	err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req,
1102 						    &write_rsp);
1103 	if (err) {
1104 		rvu_mcam_remove_counter_from_rule(rvu, owner, rule);
1105 		if (new)
1106 			kfree(rule);
1107 		return err;
1108 	}
1109 	/* update rule */
1110 	memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet));
1111 	memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask));
1112 	rule->entry = entry_index;
1113 	memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action));
1114 	if (is_npc_intf_tx(req->intf))
1115 		memcpy(&rule->tx_action, &entry->action,
1116 		       sizeof(struct nix_tx_action));
1117 	rule->vtag_action = entry->vtag_action;
1118 	rule->features = installed_features;
1119 	rule->default_rule = req->default_rule;
1120 	rule->owner = owner;
1121 	rule->enable = enable;
1122 	if (is_npc_intf_tx(req->intf))
1123 		rule->intf = pfvf->nix_tx_intf;
1124 	else
1125 		rule->intf = pfvf->nix_rx_intf;
1126 
1127 	if (new)
1128 		rvu_mcam_add_rule(mcam, rule);
1129 	if (req->default_rule)
1130 		pfvf->def_ucast_rule = rule;
1131 
1132 	/* VF's MAC address is being changed via PF  */
1133 	if (pf_set_vfs_mac) {
1134 		ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1135 		ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1136 		set_bit(PF_SET_VF_MAC, &pfvf->flags);
1137 	}
1138 
1139 	if (test_bit(PF_SET_VF_CFG, &pfvf->flags) &&
1140 	    req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7)
1141 		rule->vfvlan_cfg = true;
1142 
1143 	if (is_npc_intf_rx(req->intf) && req->match_id &&
1144 	    (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS))
1145 		return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc,
1146 					     req->index, req->match_id);
1147 
1148 	return 0;
1149 }
1150 
1151 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu,
1152 				      struct npc_install_flow_req *req,
1153 				      struct npc_install_flow_rsp *rsp)
1154 {
1155 	bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK);
1156 	struct rvu_switch *rswitch = &rvu->rswitch;
1157 	int blkaddr, nixlf, err;
1158 	struct rvu_pfvf *pfvf;
1159 	bool pf_set_vfs_mac = false;
1160 	bool enable = true;
1161 	u16 target;
1162 
1163 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1164 	if (blkaddr < 0) {
1165 		dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
1166 		return NPC_MCAM_INVALID_REQ;
1167 	}
1168 
1169 	if (!is_npc_interface_valid(rvu, req->intf))
1170 		return NPC_FLOW_INTF_INVALID;
1171 
1172 	if (from_vf && req->default_rule)
1173 		return NPC_FLOW_VF_PERM_DENIED;
1174 
1175 	/* Each PF/VF info is maintained in struct rvu_pfvf.
1176 	 * rvu_pfvf for the target PF/VF needs to be retrieved
1177 	 * hence modify pcifunc accordingly.
1178 	 */
1179 
1180 	/* AF installing for a PF/VF */
1181 	if (!req->hdr.pcifunc)
1182 		target = req->vf;
1183 	/* PF installing for its VF */
1184 	else if (!from_vf && req->vf) {
1185 		target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf;
1186 		pf_set_vfs_mac = req->default_rule &&
1187 				(req->features & BIT_ULL(NPC_DMAC));
1188 	}
1189 	/* msg received from PF/VF */
1190 	else
1191 		target = req->hdr.pcifunc;
1192 
1193 	/* ignore chan_mask in case pf func is not AF, revisit later */
1194 	if (!is_pffunc_af(req->hdr.pcifunc))
1195 		req->chan_mask = 0xFFF;
1196 
1197 	err = npc_check_unsupported_flows(rvu, req->features, req->intf);
1198 	if (err)
1199 		return NPC_FLOW_NOT_SUPPORTED;
1200 
1201 	pfvf = rvu_get_pfvf(rvu, target);
1202 
1203 	/* PF installing for its VF */
1204 	if (req->hdr.pcifunc && !from_vf && req->vf)
1205 		set_bit(PF_SET_VF_CFG, &pfvf->flags);
1206 
1207 	/* update req destination mac addr */
1208 	if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) &&
1209 	    is_zero_ether_addr(req->packet.dmac)) {
1210 		ether_addr_copy(req->packet.dmac, pfvf->mac_addr);
1211 		eth_broadcast_addr((u8 *)&req->mask.dmac);
1212 	}
1213 
1214 	/* Proceed if NIXLF is attached or not for TX rules */
1215 	err = nix_get_nixlf(rvu, target, &nixlf, NULL);
1216 	if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac)
1217 		return NPC_FLOW_NO_NIXLF;
1218 
1219 	/* don't enable rule when nixlf not attached or initialized */
1220 	if (!(is_nixlf_attached(rvu, target) &&
1221 	      test_bit(NIXLF_INITIALIZED, &pfvf->flags)))
1222 		enable = false;
1223 
1224 	/* Packets reaching NPC in Tx path implies that a
1225 	 * NIXLF is properly setup and transmitting.
1226 	 * Hence rules can be enabled for Tx.
1227 	 */
1228 	if (is_npc_intf_tx(req->intf))
1229 		enable = true;
1230 
1231 	/* Do not allow requests from uninitialized VFs */
1232 	if (from_vf && !enable)
1233 		return NPC_FLOW_VF_NOT_INIT;
1234 
1235 	/* PF sets VF mac & VF NIXLF is not attached, update the mac addr */
1236 	if (pf_set_vfs_mac && !enable) {
1237 		ether_addr_copy(pfvf->default_mac, req->packet.dmac);
1238 		ether_addr_copy(pfvf->mac_addr, req->packet.dmac);
1239 		set_bit(PF_SET_VF_MAC, &pfvf->flags);
1240 		return 0;
1241 	}
1242 
1243 	mutex_lock(&rswitch->switch_lock);
1244 	err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf,
1245 			       req, rsp, enable, pf_set_vfs_mac);
1246 	mutex_unlock(&rswitch->switch_lock);
1247 
1248 	return err;
1249 }
1250 
1251 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule,
1252 			   u16 pcifunc)
1253 {
1254 	struct npc_mcam_ena_dis_entry_req dis_req = { 0 };
1255 	struct msg_rsp dis_rsp;
1256 
1257 	if (rule->default_rule)
1258 		return 0;
1259 
1260 	if (rule->has_cntr)
1261 		rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule);
1262 
1263 	dis_req.hdr.pcifunc = pcifunc;
1264 	dis_req.entry = rule->entry;
1265 
1266 	list_del(&rule->list);
1267 	kfree(rule);
1268 
1269 	return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp);
1270 }
1271 
1272 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu,
1273 				     struct npc_delete_flow_req *req,
1274 				     struct msg_rsp *rsp)
1275 {
1276 	struct npc_mcam *mcam = &rvu->hw->mcam;
1277 	struct rvu_npc_mcam_rule *iter, *tmp;
1278 	u16 pcifunc = req->hdr.pcifunc;
1279 	struct list_head del_list;
1280 
1281 	INIT_LIST_HEAD(&del_list);
1282 
1283 	mutex_lock(&mcam->lock);
1284 	list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) {
1285 		if (iter->owner == pcifunc) {
1286 			/* All rules */
1287 			if (req->all) {
1288 				list_move_tail(&iter->list, &del_list);
1289 			/* Range of rules */
1290 			} else if (req->end && iter->entry >= req->start &&
1291 				   iter->entry <= req->end) {
1292 				list_move_tail(&iter->list, &del_list);
1293 			/* single rule */
1294 			} else if (req->entry == iter->entry) {
1295 				list_move_tail(&iter->list, &del_list);
1296 				break;
1297 			}
1298 		}
1299 	}
1300 	mutex_unlock(&mcam->lock);
1301 
1302 	list_for_each_entry_safe(iter, tmp, &del_list, list) {
1303 		u16 entry = iter->entry;
1304 
1305 		/* clear the mcam entry target pcifunc */
1306 		mcam->entry2target_pffunc[entry] = 0x0;
1307 		if (npc_delete_flow(rvu, iter, pcifunc))
1308 			dev_err(rvu->dev, "rule deletion failed for entry:%u",
1309 				entry);
1310 	}
1311 
1312 	return 0;
1313 }
1314 
1315 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr,
1316 				 struct rvu_npc_mcam_rule *rule,
1317 				 struct rvu_pfvf *pfvf)
1318 {
1319 	struct npc_mcam_write_entry_req write_req = { 0 };
1320 	struct mcam_entry *entry = &write_req.entry_data;
1321 	struct npc_mcam *mcam = &rvu->hw->mcam;
1322 	struct msg_rsp rsp;
1323 	u8 intf, enable;
1324 	int err;
1325 
1326 	ether_addr_copy(rule->packet.dmac, pfvf->mac_addr);
1327 
1328 	npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry,
1329 			    entry, &intf,  &enable);
1330 
1331 	npc_update_entry(rvu, NPC_DMAC, entry,
1332 			 ether_addr_to_u64(pfvf->mac_addr), 0,
1333 			 0xffffffffffffull, 0, intf);
1334 
1335 	write_req.hdr.pcifunc = rule->owner;
1336 	write_req.entry = rule->entry;
1337 	write_req.intf = pfvf->nix_rx_intf;
1338 
1339 	mutex_unlock(&mcam->lock);
1340 	err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp);
1341 	mutex_lock(&mcam->lock);
1342 
1343 	return err;
1344 }
1345 
1346 void npc_mcam_enable_flows(struct rvu *rvu, u16 target)
1347 {
1348 	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target);
1349 	struct rvu_npc_mcam_rule *def_ucast_rule;
1350 	struct npc_mcam *mcam = &rvu->hw->mcam;
1351 	struct rvu_npc_mcam_rule *rule;
1352 	int blkaddr, bank, index;
1353 	u64 def_action;
1354 
1355 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1356 	if (blkaddr < 0)
1357 		return;
1358 
1359 	def_ucast_rule = pfvf->def_ucast_rule;
1360 
1361 	mutex_lock(&mcam->lock);
1362 	list_for_each_entry(rule, &mcam->mcam_rules, list) {
1363 		if (is_npc_intf_rx(rule->intf) &&
1364 		    rule->rx_action.pf_func == target && !rule->enable) {
1365 			if (rule->default_rule) {
1366 				npc_enable_mcam_entry(rvu, mcam, blkaddr,
1367 						      rule->entry, true);
1368 				rule->enable = true;
1369 				continue;
1370 			}
1371 
1372 			if (rule->vfvlan_cfg)
1373 				npc_update_dmac_value(rvu, blkaddr, rule, pfvf);
1374 
1375 			if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) {
1376 				if (!def_ucast_rule)
1377 					continue;
1378 				/* Use default unicast entry action */
1379 				rule->rx_action = def_ucast_rule->rx_action;
1380 				def_action = *(u64 *)&def_ucast_rule->rx_action;
1381 				bank = npc_get_bank(mcam, rule->entry);
1382 				rvu_write64(rvu, blkaddr,
1383 					    NPC_AF_MCAMEX_BANKX_ACTION
1384 					    (rule->entry, bank), def_action);
1385 			}
1386 
1387 			npc_enable_mcam_entry(rvu, mcam, blkaddr,
1388 					      rule->entry, true);
1389 			rule->enable = true;
1390 		}
1391 	}
1392 
1393 	/* Enable MCAM entries installed by PF with target as VF pcifunc */
1394 	for (index = 0; index < mcam->bmap_entries; index++) {
1395 		if (mcam->entry2target_pffunc[index] == target)
1396 			npc_enable_mcam_entry(rvu, mcam, blkaddr,
1397 					      index, true);
1398 	}
1399 	mutex_unlock(&mcam->lock);
1400 }
1401 
1402 void npc_mcam_disable_flows(struct rvu *rvu, u16 target)
1403 {
1404 	struct npc_mcam *mcam = &rvu->hw->mcam;
1405 	int blkaddr, index;
1406 
1407 	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
1408 	if (blkaddr < 0)
1409 		return;
1410 
1411 	mutex_lock(&mcam->lock);
1412 	/* Disable MCAM entries installed by PF with target as VF pcifunc */
1413 	for (index = 0; index < mcam->bmap_entries; index++) {
1414 		if (mcam->entry2target_pffunc[index] == target)
1415 			npc_enable_mcam_entry(rvu, mcam, blkaddr,
1416 					      index, false);
1417 	}
1418 	mutex_unlock(&mcam->lock);
1419 }
1420