1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 */ 6 7 #include <linux/bitfield.h> 8 9 #include "rvu_struct.h" 10 #include "rvu_reg.h" 11 #include "rvu.h" 12 #include "npc.h" 13 14 #define NPC_BYTESM GENMASK_ULL(19, 16) 15 #define NPC_HDR_OFFSET GENMASK_ULL(15, 8) 16 #define NPC_KEY_OFFSET GENMASK_ULL(5, 0) 17 #define NPC_LDATA_EN BIT_ULL(7) 18 19 static const char * const npc_flow_names[] = { 20 [NPC_DMAC] = "dmac", 21 [NPC_SMAC] = "smac", 22 [NPC_ETYPE] = "ether type", 23 [NPC_OUTER_VID] = "outer vlan id", 24 [NPC_TOS] = "tos", 25 [NPC_SIP_IPV4] = "ipv4 source ip", 26 [NPC_DIP_IPV4] = "ipv4 destination ip", 27 [NPC_SIP_IPV6] = "ipv6 source ip", 28 [NPC_DIP_IPV6] = "ipv6 destination ip", 29 [NPC_IPPROTO_TCP] = "ip proto tcp", 30 [NPC_IPPROTO_UDP] = "ip proto udp", 31 [NPC_IPPROTO_SCTP] = "ip proto sctp", 32 [NPC_IPPROTO_ICMP] = "ip proto icmp", 33 [NPC_IPPROTO_ICMP6] = "ip proto icmp6", 34 [NPC_IPPROTO_AH] = "ip proto AH", 35 [NPC_IPPROTO_ESP] = "ip proto ESP", 36 [NPC_SPORT_TCP] = "tcp source port", 37 [NPC_DPORT_TCP] = "tcp destination port", 38 [NPC_SPORT_UDP] = "udp source port", 39 [NPC_DPORT_UDP] = "udp destination port", 40 [NPC_SPORT_SCTP] = "sctp source port", 41 [NPC_DPORT_SCTP] = "sctp destination port", 42 [NPC_UNKNOWN] = "unknown", 43 }; 44 45 const char *npc_get_field_name(u8 hdr) 46 { 47 if (hdr >= ARRAY_SIZE(npc_flow_names)) 48 return npc_flow_names[NPC_UNKNOWN]; 49 50 return npc_flow_names[hdr]; 51 } 52 53 /* Compute keyword masks and figure out the number of keywords a field 54 * spans in the key. 55 */ 56 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, 57 u8 nr_bits, int start_kwi, int offset, u8 intf) 58 { 59 struct npc_key_field *field = &mcam->rx_key_fields[type]; 60 u8 bits_in_kw; 61 int max_kwi; 62 63 if (mcam->banks_per_entry == 1) 64 max_kwi = 1; /* NPC_MCAM_KEY_X1 */ 65 else if (mcam->banks_per_entry == 2) 66 max_kwi = 3; /* NPC_MCAM_KEY_X2 */ 67 else 68 max_kwi = 6; /* NPC_MCAM_KEY_X4 */ 69 70 if (is_npc_intf_tx(intf)) 71 field = &mcam->tx_key_fields[type]; 72 73 if (offset + nr_bits <= 64) { 74 /* one KW only */ 75 if (start_kwi > max_kwi) 76 return; 77 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0) 78 << offset; 79 field->nr_kws = 1; 80 } else if (offset + nr_bits > 64 && 81 offset + nr_bits <= 128) { 82 /* two KWs */ 83 if (start_kwi + 1 > max_kwi) 84 return; 85 /* first KW mask */ 86 bits_in_kw = 64 - offset; 87 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 88 << offset; 89 /* second KW mask i.e. mask for rest of bits */ 90 bits_in_kw = nr_bits + offset - 64; 91 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0); 92 field->nr_kws = 2; 93 } else { 94 /* three KWs */ 95 if (start_kwi + 2 > max_kwi) 96 return; 97 /* first KW mask */ 98 bits_in_kw = 64 - offset; 99 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 100 << offset; 101 /* second KW mask */ 102 field->kw_mask[start_kwi + 1] = ~0ULL; 103 /* third KW mask i.e. mask for rest of bits */ 104 bits_in_kw = nr_bits + offset - 128; 105 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0); 106 field->nr_kws = 3; 107 } 108 } 109 110 /* Helper function to figure out whether field exists in the key */ 111 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf) 112 { 113 struct npc_mcam *mcam = &rvu->hw->mcam; 114 struct npc_key_field *input; 115 116 input = &mcam->rx_key_fields[type]; 117 if (is_npc_intf_tx(intf)) 118 input = &mcam->tx_key_fields[type]; 119 120 return input->nr_kws > 0; 121 } 122 123 static bool npc_is_same(struct npc_key_field *input, 124 struct npc_key_field *field) 125 { 126 return memcmp(&input->layer_mdata, &field->layer_mdata, 127 sizeof(struct npc_layer_mdata)) == 0; 128 } 129 130 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, 131 u64 cfg, u8 lid, u8 lt, u8 intf) 132 { 133 struct npc_key_field *input = &mcam->rx_key_fields[type]; 134 135 if (is_npc_intf_tx(intf)) 136 input = &mcam->tx_key_fields[type]; 137 138 input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 139 input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); 140 input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; 141 input->layer_mdata.ltype = lt; 142 input->layer_mdata.lid = lid; 143 } 144 145 static bool npc_check_overlap_fields(struct npc_key_field *input1, 146 struct npc_key_field *input2) 147 { 148 int kwi; 149 150 /* Fields with same layer id and different ltypes are mutually 151 * exclusive hence they can be overlapped 152 */ 153 if (input1->layer_mdata.lid == input2->layer_mdata.lid && 154 input1->layer_mdata.ltype != input2->layer_mdata.ltype) 155 return false; 156 157 for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { 158 if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) 159 return true; 160 } 161 162 return false; 163 } 164 165 /* Helper function to check whether given field overlaps with any other fields 166 * in the key. Due to limitations on key size and the key extraction profile in 167 * use higher layers can overwrite lower layer's header fields. Hence overlap 168 * needs to be checked. 169 */ 170 static bool npc_check_overlap(struct rvu *rvu, int blkaddr, 171 enum key_fields type, u8 start_lid, u8 intf) 172 { 173 struct npc_mcam *mcam = &rvu->hw->mcam; 174 struct npc_key_field *dummy, *input; 175 int start_kwi, offset; 176 u8 nr_bits, lid, lt, ld; 177 u64 cfg; 178 179 dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; 180 input = &mcam->rx_key_fields[type]; 181 182 if (is_npc_intf_tx(intf)) { 183 dummy = &mcam->tx_key_fields[NPC_UNKNOWN]; 184 input = &mcam->tx_key_fields[type]; 185 } 186 187 for (lid = start_lid; lid < NPC_MAX_LID; lid++) { 188 for (lt = 0; lt < NPC_MAX_LT; lt++) { 189 for (ld = 0; ld < NPC_MAX_LD; ld++) { 190 cfg = rvu_read64(rvu, blkaddr, 191 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 192 (intf, lid, lt, ld)); 193 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 194 continue; 195 memset(dummy, 0, sizeof(struct npc_key_field)); 196 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, 197 lid, lt, intf); 198 /* exclude input */ 199 if (npc_is_same(input, dummy)) 200 continue; 201 start_kwi = dummy->layer_mdata.key / 8; 202 offset = (dummy->layer_mdata.key * 8) % 64; 203 nr_bits = dummy->layer_mdata.len * 8; 204 /* form KW masks */ 205 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, 206 start_kwi, offset, intf); 207 /* check any input field bits falls in any 208 * other field bits. 209 */ 210 if (npc_check_overlap_fields(dummy, input)) 211 return true; 212 } 213 } 214 } 215 216 return false; 217 } 218 219 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, 220 u8 intf) 221 { 222 if (!npc_is_field_present(rvu, type, intf) || 223 npc_check_overlap(rvu, blkaddr, type, 0, intf)) 224 return false; 225 return true; 226 } 227 228 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, 229 u8 key_nibble, u8 intf) 230 { 231 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 232 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 233 u8 nr_bits = 4; /* bits in a nibble */ 234 u8 type; 235 236 switch (bit_number) { 237 case 0 ... 2: 238 type = NPC_CHAN; 239 break; 240 case 3: 241 type = NPC_ERRLEV; 242 break; 243 case 4 ... 5: 244 type = NPC_ERRCODE; 245 break; 246 case 6: 247 type = NPC_LXMB; 248 break; 249 /* check for LTYPE only as of now */ 250 case 9: 251 type = NPC_LA; 252 break; 253 case 12: 254 type = NPC_LB; 255 break; 256 case 15: 257 type = NPC_LC; 258 break; 259 case 18: 260 type = NPC_LD; 261 break; 262 case 21: 263 type = NPC_LE; 264 break; 265 case 24: 266 type = NPC_LF; 267 break; 268 case 27: 269 type = NPC_LG; 270 break; 271 case 30: 272 type = NPC_LH; 273 break; 274 default: 275 return; 276 } 277 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 278 } 279 280 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) 281 { 282 struct npc_mcam *mcam = &rvu->hw->mcam; 283 struct npc_key_field *key_fields; 284 /* Ether type can come from three layers 285 * (ethernet, single tagged, double tagged) 286 */ 287 struct npc_key_field *etype_ether; 288 struct npc_key_field *etype_tag1; 289 struct npc_key_field *etype_tag2; 290 /* Outer VLAN TCI can come from two layers 291 * (single tagged, double tagged) 292 */ 293 struct npc_key_field *vlan_tag1; 294 struct npc_key_field *vlan_tag2; 295 u64 *features; 296 u8 start_lid; 297 int i; 298 299 key_fields = mcam->rx_key_fields; 300 features = &mcam->rx_features; 301 302 if (is_npc_intf_tx(intf)) { 303 key_fields = mcam->tx_key_fields; 304 features = &mcam->tx_features; 305 } 306 307 /* Handle header fields which can come from multiple layers like 308 * etype, outer vlan tci. These fields should have same position in 309 * the key otherwise to install a mcam rule more than one entry is 310 * needed which complicates mcam space management. 311 */ 312 etype_ether = &key_fields[NPC_ETYPE_ETHER]; 313 etype_tag1 = &key_fields[NPC_ETYPE_TAG1]; 314 etype_tag2 = &key_fields[NPC_ETYPE_TAG2]; 315 vlan_tag1 = &key_fields[NPC_VLAN_TAG1]; 316 vlan_tag2 = &key_fields[NPC_VLAN_TAG2]; 317 318 /* if key profile programmed does not extract Ethertype at all */ 319 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 320 goto vlan_tci; 321 322 /* if key profile programmed extracts Ethertype from one layer */ 323 if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 324 key_fields[NPC_ETYPE] = *etype_ether; 325 if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws) 326 key_fields[NPC_ETYPE] = *etype_tag1; 327 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws) 328 key_fields[NPC_ETYPE] = *etype_tag2; 329 330 /* if key profile programmed extracts Ethertype from multiple layers */ 331 if (etype_ether->nr_kws && etype_tag1->nr_kws) { 332 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 333 if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) 334 goto vlan_tci; 335 } 336 key_fields[NPC_ETYPE] = *etype_tag1; 337 } 338 if (etype_ether->nr_kws && etype_tag2->nr_kws) { 339 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 340 if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) 341 goto vlan_tci; 342 } 343 key_fields[NPC_ETYPE] = *etype_tag2; 344 } 345 if (etype_tag1->nr_kws && etype_tag2->nr_kws) { 346 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 347 if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) 348 goto vlan_tci; 349 } 350 key_fields[NPC_ETYPE] = *etype_tag2; 351 } 352 353 /* check none of higher layers overwrite Ethertype */ 354 start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1; 355 if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf)) 356 goto vlan_tci; 357 *features |= BIT_ULL(NPC_ETYPE); 358 vlan_tci: 359 /* if key profile does not extract outer vlan tci at all */ 360 if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 361 goto done; 362 363 /* if key profile extracts outer vlan tci from one layer */ 364 if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 365 key_fields[NPC_OUTER_VID] = *vlan_tag1; 366 if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws) 367 key_fields[NPC_OUTER_VID] = *vlan_tag2; 368 369 /* if key profile extracts outer vlan tci from multiple layers */ 370 if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { 371 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 372 if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) 373 goto done; 374 } 375 key_fields[NPC_OUTER_VID] = *vlan_tag2; 376 } 377 /* check none of higher layers overwrite outer vlan tci */ 378 start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1; 379 if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf)) 380 goto done; 381 *features |= BIT_ULL(NPC_OUTER_VID); 382 done: 383 return; 384 } 385 386 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, 387 u8 lt, u64 cfg, u8 intf) 388 { 389 struct npc_mcam *mcam = &rvu->hw->mcam; 390 u8 hdr, key, nr_bytes, bit_offset; 391 u8 la_ltype, la_start; 392 /* starting KW index and starting bit position */ 393 int start_kwi, offset; 394 395 nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; 396 hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 397 key = FIELD_GET(NPC_KEY_OFFSET, cfg); 398 start_kwi = key / 8; 399 offset = (key * 8) % 64; 400 401 /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding 402 * ethernet header. 403 */ 404 if (is_npc_intf_tx(intf)) { 405 la_ltype = NPC_LT_LA_IH_NIX_ETHER; 406 la_start = 8; 407 } else { 408 la_ltype = NPC_LT_LA_ETHER; 409 la_start = 0; 410 } 411 412 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen) \ 413 do { \ 414 if (lid == (hlid) && lt == (hlt)) { \ 415 if ((hstart) >= hdr && \ 416 ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ 417 bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ 418 npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ 419 npc_set_kw_masks(mcam, (name), (hlen) * 8, \ 420 start_kwi, offset + bit_offset, intf);\ 421 } \ 422 } \ 423 } while (0) 424 425 /* List LID, LTYPE, start offset from layer and length(in bytes) of 426 * packet header fields below. 427 * Example: Source IP is 4 bytes and starts at 12th byte of IP header 428 */ 429 NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1); 430 NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4); 431 NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4); 432 NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16); 433 NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16); 434 NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2); 435 NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2); 436 NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2); 437 NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2); 438 NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2); 439 NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2); 440 NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2); 441 NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2); 442 NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2); 443 NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); 444 NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); 445 NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); 446 NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6); 447 /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ 448 NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2); 449 } 450 451 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) 452 { 453 struct npc_mcam *mcam = &rvu->hw->mcam; 454 u64 *features = &mcam->rx_features; 455 u64 tcp_udp_sctp; 456 int hdr; 457 458 if (is_npc_intf_tx(intf)) 459 features = &mcam->tx_features; 460 461 for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) { 462 if (npc_check_field(rvu, blkaddr, hdr, intf)) 463 *features |= BIT_ULL(hdr); 464 } 465 466 tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) | 467 BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) | 468 BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP); 469 470 /* for tcp/udp/sctp corresponding layer type should be in the key */ 471 if (*features & tcp_udp_sctp) { 472 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf)) 473 *features &= ~tcp_udp_sctp; 474 else 475 *features |= BIT_ULL(NPC_IPPROTO_TCP) | 476 BIT_ULL(NPC_IPPROTO_UDP) | 477 BIT_ULL(NPC_IPPROTO_SCTP); 478 } 479 480 /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */ 481 if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) { 482 *features |= BIT_ULL(NPC_IPPROTO_AH); 483 *features |= BIT_ULL(NPC_IPPROTO_ICMP); 484 *features |= BIT_ULL(NPC_IPPROTO_ICMP6); 485 } 486 487 /* for ESP, check if corresponding layer type is present in the key */ 488 if (npc_check_field(rvu, blkaddr, NPC_LE, intf)) 489 *features |= BIT_ULL(NPC_IPPROTO_ESP); 490 491 /* for vlan corresponding layer type should be in the key */ 492 if (*features & BIT_ULL(NPC_OUTER_VID)) 493 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) 494 *features &= ~BIT_ULL(NPC_OUTER_VID); 495 } 496 497 /* Scan key extraction profile and record how fields of our interest 498 * fill the key structure. Also verify Channel and DMAC exists in 499 * key and not overwritten by other header fields. 500 */ 501 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) 502 { 503 struct npc_mcam *mcam = &rvu->hw->mcam; 504 u8 lid, lt, ld, bitnr; 505 u8 key_nibble = 0; 506 u64 cfg; 507 508 /* Scan and note how parse result is going to be in key. 509 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from 510 * parse result in the key. The enabled nibbles from parse result 511 * will be concatenated in key. 512 */ 513 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 514 cfg &= NPC_PARSE_NIBBLE; 515 for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) { 516 npc_scan_parse_result(mcam, bitnr, key_nibble, intf); 517 key_nibble++; 518 } 519 520 /* Scan and note how layer data is going to be in key */ 521 for (lid = 0; lid < NPC_MAX_LID; lid++) { 522 for (lt = 0; lt < NPC_MAX_LT; lt++) { 523 for (ld = 0; ld < NPC_MAX_LD; ld++) { 524 cfg = rvu_read64(rvu, blkaddr, 525 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 526 (intf, lid, lt, ld)); 527 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 528 continue; 529 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, 530 intf); 531 } 532 } 533 } 534 535 return 0; 536 } 537 538 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) 539 { 540 int err; 541 542 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX); 543 if (err) 544 return err; 545 546 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX); 547 if (err) 548 return err; 549 550 /* Channel is mandatory */ 551 if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) { 552 dev_err(rvu->dev, "Channel not present in Key\n"); 553 return -EINVAL; 554 } 555 /* check that none of the fields overwrite channel */ 556 if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) { 557 dev_err(rvu->dev, "Channel cannot be overwritten\n"); 558 return -EINVAL; 559 } 560 /* DMAC should be present in key for unicast filter to work */ 561 if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) { 562 dev_err(rvu->dev, "DMAC not present in Key\n"); 563 return -EINVAL; 564 } 565 /* check that none of the fields overwrite DMAC */ 566 if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) { 567 dev_err(rvu->dev, "DMAC cannot be overwritten\n"); 568 return -EINVAL; 569 } 570 571 npc_set_features(rvu, blkaddr, NIX_INTF_TX); 572 npc_set_features(rvu, blkaddr, NIX_INTF_RX); 573 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX); 574 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX); 575 576 return 0; 577 } 578 579 int npc_flow_steering_init(struct rvu *rvu, int blkaddr) 580 { 581 struct npc_mcam *mcam = &rvu->hw->mcam; 582 583 INIT_LIST_HEAD(&mcam->mcam_rules); 584 585 return npc_scan_verify_kex(rvu, blkaddr); 586 } 587 588 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) 589 { 590 struct npc_mcam *mcam = &rvu->hw->mcam; 591 u64 *mcam_features = &mcam->rx_features; 592 u64 unsupported; 593 u8 bit; 594 595 if (is_npc_intf_tx(intf)) 596 mcam_features = &mcam->tx_features; 597 598 unsupported = (*mcam_features ^ features) & ~(*mcam_features); 599 if (unsupported) { 600 dev_info(rvu->dev, "Unsupported flow(s):\n"); 601 for_each_set_bit(bit, (unsigned long *)&unsupported, 64) 602 dev_info(rvu->dev, "%s ", npc_get_field_name(bit)); 603 return -EOPNOTSUPP; 604 } 605 606 return 0; 607 } 608 609 /* npc_update_entry - Based on the masks generated during 610 * the key scanning, updates the given entry with value and 611 * masks for the field of interest. Maximum 16 bytes of a packet 612 * header can be extracted by HW hence lo and hi are sufficient. 613 * When field bytes are less than or equal to 8 then hi should be 614 * 0 for value and mask. 615 * 616 * If exact match of value is required then mask should be all 1's. 617 * If any bits in mask are 0 then corresponding bits in value are 618 * dont care. 619 */ 620 static void npc_update_entry(struct rvu *rvu, enum key_fields type, 621 struct mcam_entry *entry, u64 val_lo, 622 u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) 623 { 624 struct npc_mcam *mcam = &rvu->hw->mcam; 625 struct mcam_entry dummy = { {0} }; 626 struct npc_key_field *field; 627 u64 kw1, kw2, kw3; 628 u8 shift; 629 int i; 630 631 field = &mcam->rx_key_fields[type]; 632 if (is_npc_intf_tx(intf)) 633 field = &mcam->tx_key_fields[type]; 634 635 if (!field->nr_kws) 636 return; 637 638 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 639 if (!field->kw_mask[i]) 640 continue; 641 /* place key value in kw[x] */ 642 shift = __ffs64(field->kw_mask[i]); 643 /* update entry value */ 644 kw1 = (val_lo << shift) & field->kw_mask[i]; 645 dummy.kw[i] = kw1; 646 /* update entry mask */ 647 kw1 = (mask_lo << shift) & field->kw_mask[i]; 648 dummy.kw_mask[i] = kw1; 649 650 if (field->nr_kws == 1) 651 break; 652 /* place remaining bits of key value in kw[x + 1] */ 653 if (field->nr_kws == 2) { 654 /* update entry value */ 655 kw2 = shift ? val_lo >> (64 - shift) : 0; 656 kw2 |= (val_hi << shift); 657 kw2 &= field->kw_mask[i + 1]; 658 dummy.kw[i + 1] = kw2; 659 /* update entry mask */ 660 kw2 = shift ? mask_lo >> (64 - shift) : 0; 661 kw2 |= (mask_hi << shift); 662 kw2 &= field->kw_mask[i + 1]; 663 dummy.kw_mask[i + 1] = kw2; 664 break; 665 } 666 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ 667 if (field->nr_kws == 3) { 668 /* update entry value */ 669 kw2 = shift ? val_lo >> (64 - shift) : 0; 670 kw2 |= (val_hi << shift); 671 kw2 &= field->kw_mask[i + 1]; 672 kw3 = shift ? val_hi >> (64 - shift) : 0; 673 kw3 &= field->kw_mask[i + 2]; 674 dummy.kw[i + 1] = kw2; 675 dummy.kw[i + 2] = kw3; 676 /* update entry mask */ 677 kw2 = shift ? mask_lo >> (64 - shift) : 0; 678 kw2 |= (mask_hi << shift); 679 kw2 &= field->kw_mask[i + 1]; 680 kw3 = shift ? mask_hi >> (64 - shift) : 0; 681 kw3 &= field->kw_mask[i + 2]; 682 dummy.kw_mask[i + 1] = kw2; 683 dummy.kw_mask[i + 2] = kw3; 684 break; 685 } 686 } 687 /* dummy is ready with values and masks for given key 688 * field now clear and update input entry with those 689 */ 690 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 691 if (!field->kw_mask[i]) 692 continue; 693 entry->kw[i] &= ~field->kw_mask[i]; 694 entry->kw_mask[i] &= ~field->kw_mask[i]; 695 696 entry->kw[i] |= dummy.kw[i]; 697 entry->kw_mask[i] |= dummy.kw_mask[i]; 698 } 699 } 700 701 #define IPV6_WORDS 4 702 703 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, 704 u64 features, struct flow_msg *pkt, 705 struct flow_msg *mask, 706 struct rvu_npc_mcam_rule *output, u8 intf) 707 { 708 u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS]; 709 u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS]; 710 struct flow_msg *opkt = &output->packet; 711 struct flow_msg *omask = &output->mask; 712 u64 mask_lo, mask_hi; 713 u64 val_lo, val_hi; 714 715 /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet 716 * values to be programmed in MCAM should as below: 717 * val_high: 0xfe80000000000000 718 * val_low: 0x2c6863fffe5e2d0a 719 */ 720 if (features & BIT_ULL(NPC_SIP_IPV6)) { 721 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS); 722 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); 723 724 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1]; 725 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3]; 726 val_hi = (u64)src_ip[0] << 32 | src_ip[1]; 727 val_lo = (u64)src_ip[2] << 32 | src_ip[3]; 728 729 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, 730 mask_lo, mask_hi, intf); 731 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); 732 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); 733 } 734 if (features & BIT_ULL(NPC_DIP_IPV6)) { 735 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS); 736 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); 737 738 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1]; 739 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3]; 740 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; 741 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; 742 743 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, 744 mask_lo, mask_hi, intf); 745 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); 746 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); 747 } 748 } 749 750 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, 751 u64 features, struct flow_msg *pkt, 752 struct flow_msg *mask, 753 struct rvu_npc_mcam_rule *output, u8 intf) 754 { 755 u64 dmac_mask = ether_addr_to_u64(mask->dmac); 756 u64 smac_mask = ether_addr_to_u64(mask->smac); 757 u64 dmac_val = ether_addr_to_u64(pkt->dmac); 758 u64 smac_val = ether_addr_to_u64(pkt->smac); 759 struct flow_msg *opkt = &output->packet; 760 struct flow_msg *omask = &output->mask; 761 762 if (!features) 763 return; 764 765 /* For tcp/udp/sctp LTYPE should be present in entry */ 766 if (features & BIT_ULL(NPC_IPPROTO_TCP)) 767 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, 768 0, ~0ULL, 0, intf); 769 if (features & BIT_ULL(NPC_IPPROTO_UDP)) 770 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, 771 0, ~0ULL, 0, intf); 772 if (features & BIT_ULL(NPC_IPPROTO_SCTP)) 773 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, 774 0, ~0ULL, 0, intf); 775 if (features & BIT_ULL(NPC_IPPROTO_ICMP)) 776 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, 777 0, ~0ULL, 0, intf); 778 if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) 779 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, 780 0, ~0ULL, 0, intf); 781 782 if (features & BIT_ULL(NPC_OUTER_VID)) 783 npc_update_entry(rvu, NPC_LB, entry, 784 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, 785 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); 786 787 /* For AH, LTYPE should be present in entry */ 788 if (features & BIT_ULL(NPC_IPPROTO_AH)) 789 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, 790 0, ~0ULL, 0, intf); 791 /* For ESP, LTYPE should be present in entry */ 792 if (features & BIT_ULL(NPC_IPPROTO_ESP)) 793 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, 794 0, ~0ULL, 0, intf); 795 796 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ 797 do { \ 798 if (features & BIT_ULL((field))) { \ 799 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ 800 (mask_lo), (mask_hi), intf); \ 801 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ 802 memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ 803 } \ 804 } while (0) 805 806 NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0); 807 NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0); 808 NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0, 809 ntohs(mask->etype), 0); 810 NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0); 811 NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0, 812 ntohl(mask->ip4src), 0); 813 NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0, 814 ntohl(mask->ip4dst), 0); 815 NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0, 816 ntohs(mask->sport), 0); 817 NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0, 818 ntohs(mask->sport), 0); 819 NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0, 820 ntohs(mask->dport), 0); 821 NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0, 822 ntohs(mask->dport), 0); 823 NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0, 824 ntohs(mask->sport), 0); 825 NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0, 826 ntohs(mask->dport), 0); 827 828 NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0, 829 ntohs(mask->vlan_tci), 0); 830 831 npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); 832 } 833 834 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, 835 u16 entry) 836 { 837 struct rvu_npc_mcam_rule *iter; 838 839 mutex_lock(&mcam->lock); 840 list_for_each_entry(iter, &mcam->mcam_rules, list) { 841 if (iter->entry == entry) { 842 mutex_unlock(&mcam->lock); 843 return iter; 844 } 845 } 846 mutex_unlock(&mcam->lock); 847 848 return NULL; 849 } 850 851 static void rvu_mcam_add_rule(struct npc_mcam *mcam, 852 struct rvu_npc_mcam_rule *rule) 853 { 854 struct list_head *head = &mcam->mcam_rules; 855 struct rvu_npc_mcam_rule *iter; 856 857 mutex_lock(&mcam->lock); 858 list_for_each_entry(iter, &mcam->mcam_rules, list) { 859 if (iter->entry > rule->entry) 860 break; 861 head = &iter->list; 862 } 863 864 list_add(&rule->list, head); 865 mutex_unlock(&mcam->lock); 866 } 867 868 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 869 struct rvu_npc_mcam_rule *rule) 870 { 871 struct npc_mcam_oper_counter_req free_req = { 0 }; 872 struct msg_rsp free_rsp; 873 874 if (!rule->has_cntr) 875 return; 876 877 free_req.hdr.pcifunc = pcifunc; 878 free_req.cntr = rule->cntr; 879 880 rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp); 881 rule->has_cntr = false; 882 } 883 884 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 885 struct rvu_npc_mcam_rule *rule, 886 struct npc_install_flow_rsp *rsp) 887 { 888 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 889 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 890 int err; 891 892 cntr_req.hdr.pcifunc = pcifunc; 893 cntr_req.contig = true; 894 cntr_req.count = 1; 895 896 /* we try to allocate a counter to track the stats of this 897 * rule. If counter could not be allocated then proceed 898 * without counter because counters are limited than entries. 899 */ 900 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, 901 &cntr_rsp); 902 if (!err && cntr_rsp.count) { 903 rule->cntr = cntr_rsp.cntr; 904 rule->has_cntr = true; 905 rsp->counter = rule->cntr; 906 } else { 907 rsp->counter = err; 908 } 909 } 910 911 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 912 struct mcam_entry *entry, 913 struct npc_install_flow_req *req, 914 u16 target, bool pf_set_vfs_mac) 915 { 916 struct rvu_switch *rswitch = &rvu->rswitch; 917 struct nix_rx_action action; 918 919 if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) 920 req->chan_mask = 0x0; /* Do not care channel */ 921 922 npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, 923 0, NIX_INTF_RX); 924 925 *(u64 *)&action = 0x00; 926 action.pf_func = target; 927 action.op = req->op; 928 action.index = req->index; 929 action.match_id = req->match_id; 930 action.flow_key_alg = req->flow_key_alg; 931 932 if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule) 933 action = pfvf->def_ucast_rule->rx_action; 934 935 entry->action = *(u64 *)&action; 936 937 /* VTAG0 starts at 0th byte of LID_B. 938 * VTAG1 starts at 4th byte of LID_B. 939 */ 940 entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | 941 FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | 942 FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | 943 FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | 944 FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) | 945 FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) | 946 FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) | 947 FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4); 948 } 949 950 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 951 struct mcam_entry *entry, 952 struct npc_install_flow_req *req, u16 target) 953 { 954 struct nix_tx_action action; 955 u64 mask = ~0ULL; 956 957 /* If AF is installing then do not care about 958 * PF_FUNC in Send Descriptor 959 */ 960 if (is_pffunc_af(req->hdr.pcifunc)) 961 mask = 0; 962 963 npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), 964 0, mask, 0, NIX_INTF_TX); 965 966 *(u64 *)&action = 0x00; 967 action.op = req->op; 968 action.index = req->index; 969 action.match_id = req->match_id; 970 971 entry->action = *(u64 *)&action; 972 973 /* VTAG0 starts at 0th byte of LID_B. 974 * VTAG1 starts at 4th byte of LID_B. 975 */ 976 entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | 977 FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | 978 FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | 979 FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | 980 FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) | 981 FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) | 982 FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) | 983 FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24); 984 } 985 986 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, 987 int nixlf, struct rvu_pfvf *pfvf, 988 struct npc_install_flow_req *req, 989 struct npc_install_flow_rsp *rsp, bool enable, 990 bool pf_set_vfs_mac) 991 { 992 struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; 993 u64 features, installed_features, missing_features = 0; 994 struct npc_mcam_write_entry_req write_req = { 0 }; 995 struct npc_mcam *mcam = &rvu->hw->mcam; 996 struct rvu_npc_mcam_rule dummy = { 0 }; 997 struct rvu_npc_mcam_rule *rule; 998 u16 owner = req->hdr.pcifunc; 999 struct msg_rsp write_rsp; 1000 struct mcam_entry *entry; 1001 int entry_index, err; 1002 bool new = false; 1003 1004 installed_features = req->features; 1005 features = req->features; 1006 entry = &write_req.entry_data; 1007 entry_index = req->entry; 1008 1009 npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, 1010 req->intf); 1011 1012 if (is_npc_intf_rx(req->intf)) 1013 npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); 1014 else 1015 npc_update_tx_entry(rvu, pfvf, entry, req, target); 1016 1017 /* Default unicast rules do not exist for TX */ 1018 if (is_npc_intf_tx(req->intf)) 1019 goto find_rule; 1020 1021 if (req->default_rule) { 1022 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf, 1023 NIXLF_UCAST_ENTRY); 1024 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index); 1025 } 1026 1027 /* update mcam entry with default unicast rule attributes */ 1028 if (def_ucast_rule && (req->default_rule && req->append)) { 1029 missing_features = (def_ucast_rule->features ^ features) & 1030 def_ucast_rule->features; 1031 if (missing_features) 1032 npc_update_flow(rvu, entry, missing_features, 1033 &def_ucast_rule->packet, 1034 &def_ucast_rule->mask, 1035 &dummy, req->intf); 1036 installed_features = req->features | missing_features; 1037 } 1038 1039 find_rule: 1040 rule = rvu_mcam_find_rule(mcam, entry_index); 1041 if (!rule) { 1042 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1043 if (!rule) 1044 return -ENOMEM; 1045 new = true; 1046 } 1047 1048 /* allocate new counter if rule has no counter */ 1049 if (!req->default_rule && req->set_cntr && !rule->has_cntr) 1050 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); 1051 1052 /* if user wants to delete an existing counter for a rule then 1053 * free the counter 1054 */ 1055 if (!req->set_cntr && rule->has_cntr) 1056 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1057 1058 write_req.hdr.pcifunc = owner; 1059 1060 /* AF owns the default rules so change the owner just to relax 1061 * the checks in rvu_mbox_handler_npc_mcam_write_entry 1062 */ 1063 if (req->default_rule) 1064 write_req.hdr.pcifunc = 0; 1065 1066 write_req.entry = entry_index; 1067 write_req.intf = req->intf; 1068 write_req.enable_entry = (u8)enable; 1069 /* if counter is available then clear and use it */ 1070 if (req->set_cntr && rule->has_cntr) { 1071 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00); 1072 write_req.set_cntr = 1; 1073 write_req.cntr = rule->cntr; 1074 } 1075 1076 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, 1077 &write_rsp); 1078 if (err) { 1079 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1080 if (new) 1081 kfree(rule); 1082 return err; 1083 } 1084 /* update rule */ 1085 memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); 1086 memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); 1087 rule->entry = entry_index; 1088 memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); 1089 if (is_npc_intf_tx(req->intf)) 1090 memcpy(&rule->tx_action, &entry->action, 1091 sizeof(struct nix_tx_action)); 1092 rule->vtag_action = entry->vtag_action; 1093 rule->features = installed_features; 1094 rule->default_rule = req->default_rule; 1095 rule->owner = owner; 1096 rule->enable = enable; 1097 if (is_npc_intf_tx(req->intf)) 1098 rule->intf = pfvf->nix_tx_intf; 1099 else 1100 rule->intf = pfvf->nix_rx_intf; 1101 1102 if (new) 1103 rvu_mcam_add_rule(mcam, rule); 1104 if (req->default_rule) 1105 pfvf->def_ucast_rule = rule; 1106 1107 /* VF's MAC address is being changed via PF */ 1108 if (pf_set_vfs_mac) { 1109 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1110 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1111 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1112 } 1113 1114 if (test_bit(PF_SET_VF_CFG, &pfvf->flags) && 1115 req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) 1116 rule->vfvlan_cfg = true; 1117 1118 if (is_npc_intf_rx(req->intf) && req->match_id && 1119 (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS)) 1120 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc, 1121 req->index, req->match_id); 1122 1123 return 0; 1124 } 1125 1126 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, 1127 struct npc_install_flow_req *req, 1128 struct npc_install_flow_rsp *rsp) 1129 { 1130 bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK); 1131 struct rvu_switch *rswitch = &rvu->rswitch; 1132 int blkaddr, nixlf, err; 1133 struct rvu_pfvf *pfvf; 1134 bool pf_set_vfs_mac = false; 1135 bool enable = true; 1136 u16 target; 1137 1138 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1139 if (blkaddr < 0) { 1140 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1141 return NPC_MCAM_INVALID_REQ; 1142 } 1143 1144 if (!is_npc_interface_valid(rvu, req->intf)) 1145 return NPC_FLOW_INTF_INVALID; 1146 1147 if (from_vf && req->default_rule) 1148 return NPC_FLOW_VF_PERM_DENIED; 1149 1150 /* Each PF/VF info is maintained in struct rvu_pfvf. 1151 * rvu_pfvf for the target PF/VF needs to be retrieved 1152 * hence modify pcifunc accordingly. 1153 */ 1154 1155 /* AF installing for a PF/VF */ 1156 if (!req->hdr.pcifunc) 1157 target = req->vf; 1158 /* PF installing for its VF */ 1159 else if (!from_vf && req->vf) { 1160 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf; 1161 pf_set_vfs_mac = req->default_rule && 1162 (req->features & BIT_ULL(NPC_DMAC)); 1163 } 1164 /* msg received from PF/VF */ 1165 else 1166 target = req->hdr.pcifunc; 1167 1168 /* ignore chan_mask in case pf func is not AF, revisit later */ 1169 if (!is_pffunc_af(req->hdr.pcifunc)) 1170 req->chan_mask = 0xFFF; 1171 1172 err = npc_check_unsupported_flows(rvu, req->features, req->intf); 1173 if (err) 1174 return NPC_FLOW_NOT_SUPPORTED; 1175 1176 pfvf = rvu_get_pfvf(rvu, target); 1177 1178 /* PF installing for its VF */ 1179 if (req->hdr.pcifunc && !from_vf && req->vf) 1180 set_bit(PF_SET_VF_CFG, &pfvf->flags); 1181 1182 /* update req destination mac addr */ 1183 if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) && 1184 is_zero_ether_addr(req->packet.dmac)) { 1185 ether_addr_copy(req->packet.dmac, pfvf->mac_addr); 1186 eth_broadcast_addr((u8 *)&req->mask.dmac); 1187 } 1188 1189 /* Proceed if NIXLF is attached or not for TX rules */ 1190 err = nix_get_nixlf(rvu, target, &nixlf, NULL); 1191 if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) 1192 return NPC_FLOW_NO_NIXLF; 1193 1194 /* don't enable rule when nixlf not attached or initialized */ 1195 if (!(is_nixlf_attached(rvu, target) && 1196 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 1197 enable = false; 1198 1199 /* Packets reaching NPC in Tx path implies that a 1200 * NIXLF is properly setup and transmitting. 1201 * Hence rules can be enabled for Tx. 1202 */ 1203 if (is_npc_intf_tx(req->intf)) 1204 enable = true; 1205 1206 /* Do not allow requests from uninitialized VFs */ 1207 if (from_vf && !enable) 1208 return NPC_FLOW_VF_NOT_INIT; 1209 1210 /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ 1211 if (pf_set_vfs_mac && !enable) { 1212 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1213 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1214 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1215 return 0; 1216 } 1217 1218 mutex_lock(&rswitch->switch_lock); 1219 err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, 1220 req, rsp, enable, pf_set_vfs_mac); 1221 mutex_unlock(&rswitch->switch_lock); 1222 1223 return err; 1224 } 1225 1226 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule, 1227 u16 pcifunc) 1228 { 1229 struct npc_mcam_ena_dis_entry_req dis_req = { 0 }; 1230 struct msg_rsp dis_rsp; 1231 1232 if (rule->default_rule) 1233 return 0; 1234 1235 if (rule->has_cntr) 1236 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); 1237 1238 dis_req.hdr.pcifunc = pcifunc; 1239 dis_req.entry = rule->entry; 1240 1241 list_del(&rule->list); 1242 kfree(rule); 1243 1244 return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp); 1245 } 1246 1247 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, 1248 struct npc_delete_flow_req *req, 1249 struct msg_rsp *rsp) 1250 { 1251 struct npc_mcam *mcam = &rvu->hw->mcam; 1252 struct rvu_npc_mcam_rule *iter, *tmp; 1253 u16 pcifunc = req->hdr.pcifunc; 1254 struct list_head del_list; 1255 1256 INIT_LIST_HEAD(&del_list); 1257 1258 mutex_lock(&mcam->lock); 1259 list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) { 1260 if (iter->owner == pcifunc) { 1261 /* All rules */ 1262 if (req->all) { 1263 list_move_tail(&iter->list, &del_list); 1264 /* Range of rules */ 1265 } else if (req->end && iter->entry >= req->start && 1266 iter->entry <= req->end) { 1267 list_move_tail(&iter->list, &del_list); 1268 /* single rule */ 1269 } else if (req->entry == iter->entry) { 1270 list_move_tail(&iter->list, &del_list); 1271 break; 1272 } 1273 } 1274 } 1275 mutex_unlock(&mcam->lock); 1276 1277 list_for_each_entry_safe(iter, tmp, &del_list, list) { 1278 u16 entry = iter->entry; 1279 1280 /* clear the mcam entry target pcifunc */ 1281 mcam->entry2target_pffunc[entry] = 0x0; 1282 if (npc_delete_flow(rvu, iter, pcifunc)) 1283 dev_err(rvu->dev, "rule deletion failed for entry:%u", 1284 entry); 1285 } 1286 1287 return 0; 1288 } 1289 1290 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, 1291 struct rvu_npc_mcam_rule *rule, 1292 struct rvu_pfvf *pfvf) 1293 { 1294 struct npc_mcam_write_entry_req write_req = { 0 }; 1295 struct mcam_entry *entry = &write_req.entry_data; 1296 struct npc_mcam *mcam = &rvu->hw->mcam; 1297 struct msg_rsp rsp; 1298 u8 intf, enable; 1299 int err; 1300 1301 ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); 1302 1303 npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, 1304 entry, &intf, &enable); 1305 1306 npc_update_entry(rvu, NPC_DMAC, entry, 1307 ether_addr_to_u64(pfvf->mac_addr), 0, 1308 0xffffffffffffull, 0, intf); 1309 1310 write_req.hdr.pcifunc = rule->owner; 1311 write_req.entry = rule->entry; 1312 write_req.intf = pfvf->nix_rx_intf; 1313 1314 mutex_unlock(&mcam->lock); 1315 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); 1316 mutex_lock(&mcam->lock); 1317 1318 return err; 1319 } 1320 1321 void npc_mcam_enable_flows(struct rvu *rvu, u16 target) 1322 { 1323 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target); 1324 struct rvu_npc_mcam_rule *def_ucast_rule; 1325 struct npc_mcam *mcam = &rvu->hw->mcam; 1326 struct rvu_npc_mcam_rule *rule; 1327 int blkaddr, bank, index; 1328 u64 def_action; 1329 1330 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1331 if (blkaddr < 0) 1332 return; 1333 1334 def_ucast_rule = pfvf->def_ucast_rule; 1335 1336 mutex_lock(&mcam->lock); 1337 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1338 if (is_npc_intf_rx(rule->intf) && 1339 rule->rx_action.pf_func == target && !rule->enable) { 1340 if (rule->default_rule) { 1341 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1342 rule->entry, true); 1343 rule->enable = true; 1344 continue; 1345 } 1346 1347 if (rule->vfvlan_cfg) 1348 npc_update_dmac_value(rvu, blkaddr, rule, pfvf); 1349 1350 if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) { 1351 if (!def_ucast_rule) 1352 continue; 1353 /* Use default unicast entry action */ 1354 rule->rx_action = def_ucast_rule->rx_action; 1355 def_action = *(u64 *)&def_ucast_rule->rx_action; 1356 bank = npc_get_bank(mcam, rule->entry); 1357 rvu_write64(rvu, blkaddr, 1358 NPC_AF_MCAMEX_BANKX_ACTION 1359 (rule->entry, bank), def_action); 1360 } 1361 1362 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1363 rule->entry, true); 1364 rule->enable = true; 1365 } 1366 } 1367 1368 /* Enable MCAM entries installed by PF with target as VF pcifunc */ 1369 for (index = 0; index < mcam->bmap_entries; index++) { 1370 if (mcam->entry2target_pffunc[index] == target) 1371 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1372 index, true); 1373 } 1374 mutex_unlock(&mcam->lock); 1375 } 1376 1377 void npc_mcam_disable_flows(struct rvu *rvu, u16 target) 1378 { 1379 struct npc_mcam *mcam = &rvu->hw->mcam; 1380 int blkaddr, index; 1381 1382 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1383 if (blkaddr < 0) 1384 return; 1385 1386 mutex_lock(&mcam->lock); 1387 /* Disable MCAM entries installed by PF with target as VF pcifunc */ 1388 for (index = 0; index < mcam->bmap_entries; index++) { 1389 if (mcam->entry2target_pffunc[index] == target) 1390 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1391 index, false); 1392 } 1393 mutex_unlock(&mcam->lock); 1394 } 1395