1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 */ 6 7 #include <linux/bitfield.h> 8 9 #include "rvu_struct.h" 10 #include "rvu_reg.h" 11 #include "rvu.h" 12 #include "npc.h" 13 14 #define NPC_BYTESM GENMASK_ULL(19, 16) 15 #define NPC_HDR_OFFSET GENMASK_ULL(15, 8) 16 #define NPC_KEY_OFFSET GENMASK_ULL(5, 0) 17 #define NPC_LDATA_EN BIT_ULL(7) 18 19 static const char * const npc_flow_names[] = { 20 [NPC_DMAC] = "dmac", 21 [NPC_SMAC] = "smac", 22 [NPC_ETYPE] = "ether type", 23 [NPC_VLAN_ETYPE_CTAG] = "vlan ether type ctag", 24 [NPC_VLAN_ETYPE_STAG] = "vlan ether type stag", 25 [NPC_OUTER_VID] = "outer vlan id", 26 [NPC_TOS] = "tos", 27 [NPC_SIP_IPV4] = "ipv4 source ip", 28 [NPC_DIP_IPV4] = "ipv4 destination ip", 29 [NPC_SIP_IPV6] = "ipv6 source ip", 30 [NPC_DIP_IPV6] = "ipv6 destination ip", 31 [NPC_IPPROTO_TCP] = "ip proto tcp", 32 [NPC_IPPROTO_UDP] = "ip proto udp", 33 [NPC_IPPROTO_SCTP] = "ip proto sctp", 34 [NPC_IPPROTO_ICMP] = "ip proto icmp", 35 [NPC_IPPROTO_ICMP6] = "ip proto icmp6", 36 [NPC_IPPROTO_AH] = "ip proto AH", 37 [NPC_IPPROTO_ESP] = "ip proto ESP", 38 [NPC_SPORT_TCP] = "tcp source port", 39 [NPC_DPORT_TCP] = "tcp destination port", 40 [NPC_SPORT_UDP] = "udp source port", 41 [NPC_DPORT_UDP] = "udp destination port", 42 [NPC_SPORT_SCTP] = "sctp source port", 43 [NPC_DPORT_SCTP] = "sctp destination port", 44 [NPC_UNKNOWN] = "unknown", 45 }; 46 47 const char *npc_get_field_name(u8 hdr) 48 { 49 if (hdr >= ARRAY_SIZE(npc_flow_names)) 50 return npc_flow_names[NPC_UNKNOWN]; 51 52 return npc_flow_names[hdr]; 53 } 54 55 /* Compute keyword masks and figure out the number of keywords a field 56 * spans in the key. 57 */ 58 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, 59 u8 nr_bits, int start_kwi, int offset, u8 intf) 60 { 61 struct npc_key_field *field = &mcam->rx_key_fields[type]; 62 u8 bits_in_kw; 63 int max_kwi; 64 65 if (mcam->banks_per_entry == 1) 66 max_kwi = 1; /* NPC_MCAM_KEY_X1 */ 67 else if (mcam->banks_per_entry == 2) 68 max_kwi = 3; /* NPC_MCAM_KEY_X2 */ 69 else 70 max_kwi = 6; /* NPC_MCAM_KEY_X4 */ 71 72 if (is_npc_intf_tx(intf)) 73 field = &mcam->tx_key_fields[type]; 74 75 if (offset + nr_bits <= 64) { 76 /* one KW only */ 77 if (start_kwi > max_kwi) 78 return; 79 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0) 80 << offset; 81 field->nr_kws = 1; 82 } else if (offset + nr_bits > 64 && 83 offset + nr_bits <= 128) { 84 /* two KWs */ 85 if (start_kwi + 1 > max_kwi) 86 return; 87 /* first KW mask */ 88 bits_in_kw = 64 - offset; 89 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 90 << offset; 91 /* second KW mask i.e. mask for rest of bits */ 92 bits_in_kw = nr_bits + offset - 64; 93 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0); 94 field->nr_kws = 2; 95 } else { 96 /* three KWs */ 97 if (start_kwi + 2 > max_kwi) 98 return; 99 /* first KW mask */ 100 bits_in_kw = 64 - offset; 101 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 102 << offset; 103 /* second KW mask */ 104 field->kw_mask[start_kwi + 1] = ~0ULL; 105 /* third KW mask i.e. mask for rest of bits */ 106 bits_in_kw = nr_bits + offset - 128; 107 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0); 108 field->nr_kws = 3; 109 } 110 } 111 112 /* Helper function to figure out whether field exists in the key */ 113 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf) 114 { 115 struct npc_mcam *mcam = &rvu->hw->mcam; 116 struct npc_key_field *input; 117 118 input = &mcam->rx_key_fields[type]; 119 if (is_npc_intf_tx(intf)) 120 input = &mcam->tx_key_fields[type]; 121 122 return input->nr_kws > 0; 123 } 124 125 static bool npc_is_same(struct npc_key_field *input, 126 struct npc_key_field *field) 127 { 128 return memcmp(&input->layer_mdata, &field->layer_mdata, 129 sizeof(struct npc_layer_mdata)) == 0; 130 } 131 132 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, 133 u64 cfg, u8 lid, u8 lt, u8 intf) 134 { 135 struct npc_key_field *input = &mcam->rx_key_fields[type]; 136 137 if (is_npc_intf_tx(intf)) 138 input = &mcam->tx_key_fields[type]; 139 140 input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 141 input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); 142 input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; 143 input->layer_mdata.ltype = lt; 144 input->layer_mdata.lid = lid; 145 } 146 147 static bool npc_check_overlap_fields(struct npc_key_field *input1, 148 struct npc_key_field *input2) 149 { 150 int kwi; 151 152 /* Fields with same layer id and different ltypes are mutually 153 * exclusive hence they can be overlapped 154 */ 155 if (input1->layer_mdata.lid == input2->layer_mdata.lid && 156 input1->layer_mdata.ltype != input2->layer_mdata.ltype) 157 return false; 158 159 for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { 160 if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) 161 return true; 162 } 163 164 return false; 165 } 166 167 /* Helper function to check whether given field overlaps with any other fields 168 * in the key. Due to limitations on key size and the key extraction profile in 169 * use higher layers can overwrite lower layer's header fields. Hence overlap 170 * needs to be checked. 171 */ 172 static bool npc_check_overlap(struct rvu *rvu, int blkaddr, 173 enum key_fields type, u8 start_lid, u8 intf) 174 { 175 struct npc_mcam *mcam = &rvu->hw->mcam; 176 struct npc_key_field *dummy, *input; 177 int start_kwi, offset; 178 u8 nr_bits, lid, lt, ld; 179 u64 cfg; 180 181 dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; 182 input = &mcam->rx_key_fields[type]; 183 184 if (is_npc_intf_tx(intf)) { 185 dummy = &mcam->tx_key_fields[NPC_UNKNOWN]; 186 input = &mcam->tx_key_fields[type]; 187 } 188 189 for (lid = start_lid; lid < NPC_MAX_LID; lid++) { 190 for (lt = 0; lt < NPC_MAX_LT; lt++) { 191 for (ld = 0; ld < NPC_MAX_LD; ld++) { 192 cfg = rvu_read64(rvu, blkaddr, 193 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 194 (intf, lid, lt, ld)); 195 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 196 continue; 197 memset(dummy, 0, sizeof(struct npc_key_field)); 198 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, 199 lid, lt, intf); 200 /* exclude input */ 201 if (npc_is_same(input, dummy)) 202 continue; 203 start_kwi = dummy->layer_mdata.key / 8; 204 offset = (dummy->layer_mdata.key * 8) % 64; 205 nr_bits = dummy->layer_mdata.len * 8; 206 /* form KW masks */ 207 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, 208 start_kwi, offset, intf); 209 /* check any input field bits falls in any 210 * other field bits. 211 */ 212 if (npc_check_overlap_fields(dummy, input)) 213 return true; 214 } 215 } 216 } 217 218 return false; 219 } 220 221 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, 222 u8 intf) 223 { 224 if (!npc_is_field_present(rvu, type, intf) || 225 npc_check_overlap(rvu, blkaddr, type, 0, intf)) 226 return false; 227 return true; 228 } 229 230 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, 231 u8 key_nibble, u8 intf) 232 { 233 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 234 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 235 u8 nr_bits = 4; /* bits in a nibble */ 236 u8 type; 237 238 switch (bit_number) { 239 case 0 ... 2: 240 type = NPC_CHAN; 241 break; 242 case 3: 243 type = NPC_ERRLEV; 244 break; 245 case 4 ... 5: 246 type = NPC_ERRCODE; 247 break; 248 case 6: 249 type = NPC_LXMB; 250 break; 251 /* check for LTYPE only as of now */ 252 case 9: 253 type = NPC_LA; 254 break; 255 case 12: 256 type = NPC_LB; 257 break; 258 case 15: 259 type = NPC_LC; 260 break; 261 case 18: 262 type = NPC_LD; 263 break; 264 case 21: 265 type = NPC_LE; 266 break; 267 case 24: 268 type = NPC_LF; 269 break; 270 case 27: 271 type = NPC_LG; 272 break; 273 case 30: 274 type = NPC_LH; 275 break; 276 default: 277 return; 278 } 279 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 280 } 281 282 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) 283 { 284 struct npc_mcam *mcam = &rvu->hw->mcam; 285 struct npc_key_field *key_fields; 286 /* Ether type can come from three layers 287 * (ethernet, single tagged, double tagged) 288 */ 289 struct npc_key_field *etype_ether; 290 struct npc_key_field *etype_tag1; 291 struct npc_key_field *etype_tag2; 292 /* Outer VLAN TCI can come from two layers 293 * (single tagged, double tagged) 294 */ 295 struct npc_key_field *vlan_tag1; 296 struct npc_key_field *vlan_tag2; 297 u64 *features; 298 u8 start_lid; 299 int i; 300 301 key_fields = mcam->rx_key_fields; 302 features = &mcam->rx_features; 303 304 if (is_npc_intf_tx(intf)) { 305 key_fields = mcam->tx_key_fields; 306 features = &mcam->tx_features; 307 } 308 309 /* Handle header fields which can come from multiple layers like 310 * etype, outer vlan tci. These fields should have same position in 311 * the key otherwise to install a mcam rule more than one entry is 312 * needed which complicates mcam space management. 313 */ 314 etype_ether = &key_fields[NPC_ETYPE_ETHER]; 315 etype_tag1 = &key_fields[NPC_ETYPE_TAG1]; 316 etype_tag2 = &key_fields[NPC_ETYPE_TAG2]; 317 vlan_tag1 = &key_fields[NPC_VLAN_TAG1]; 318 vlan_tag2 = &key_fields[NPC_VLAN_TAG2]; 319 320 /* if key profile programmed does not extract Ethertype at all */ 321 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 322 goto vlan_tci; 323 324 /* if key profile programmed extracts Ethertype from one layer */ 325 if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 326 key_fields[NPC_ETYPE] = *etype_ether; 327 if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws) 328 key_fields[NPC_ETYPE] = *etype_tag1; 329 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws) 330 key_fields[NPC_ETYPE] = *etype_tag2; 331 332 /* if key profile programmed extracts Ethertype from multiple layers */ 333 if (etype_ether->nr_kws && etype_tag1->nr_kws) { 334 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 335 if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) 336 goto vlan_tci; 337 } 338 key_fields[NPC_ETYPE] = *etype_tag1; 339 } 340 if (etype_ether->nr_kws && etype_tag2->nr_kws) { 341 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 342 if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) 343 goto vlan_tci; 344 } 345 key_fields[NPC_ETYPE] = *etype_tag2; 346 } 347 if (etype_tag1->nr_kws && etype_tag2->nr_kws) { 348 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 349 if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) 350 goto vlan_tci; 351 } 352 key_fields[NPC_ETYPE] = *etype_tag2; 353 } 354 355 /* check none of higher layers overwrite Ethertype */ 356 start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1; 357 if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf)) 358 goto vlan_tci; 359 *features |= BIT_ULL(NPC_ETYPE); 360 vlan_tci: 361 /* if key profile does not extract outer vlan tci at all */ 362 if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 363 goto done; 364 365 /* if key profile extracts outer vlan tci from one layer */ 366 if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 367 key_fields[NPC_OUTER_VID] = *vlan_tag1; 368 if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws) 369 key_fields[NPC_OUTER_VID] = *vlan_tag2; 370 371 /* if key profile extracts outer vlan tci from multiple layers */ 372 if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { 373 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 374 if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) 375 goto done; 376 } 377 key_fields[NPC_OUTER_VID] = *vlan_tag2; 378 } 379 /* check none of higher layers overwrite outer vlan tci */ 380 start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1; 381 if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf)) 382 goto done; 383 *features |= BIT_ULL(NPC_OUTER_VID); 384 done: 385 return; 386 } 387 388 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, 389 u8 lt, u64 cfg, u8 intf) 390 { 391 struct npc_mcam *mcam = &rvu->hw->mcam; 392 u8 hdr, key, nr_bytes, bit_offset; 393 u8 la_ltype, la_start; 394 /* starting KW index and starting bit position */ 395 int start_kwi, offset; 396 397 nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; 398 hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 399 key = FIELD_GET(NPC_KEY_OFFSET, cfg); 400 start_kwi = key / 8; 401 offset = (key * 8) % 64; 402 403 /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding 404 * ethernet header. 405 */ 406 if (is_npc_intf_tx(intf)) { 407 la_ltype = NPC_LT_LA_IH_NIX_ETHER; 408 la_start = 8; 409 } else { 410 la_ltype = NPC_LT_LA_ETHER; 411 la_start = 0; 412 } 413 414 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen) \ 415 do { \ 416 if (lid == (hlid) && lt == (hlt)) { \ 417 if ((hstart) >= hdr && \ 418 ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ 419 bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ 420 npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ 421 npc_set_kw_masks(mcam, (name), (hlen) * 8, \ 422 start_kwi, offset + bit_offset, intf);\ 423 } \ 424 } \ 425 } while (0) 426 427 /* List LID, LTYPE, start offset from layer and length(in bytes) of 428 * packet header fields below. 429 * Example: Source IP is 4 bytes and starts at 12th byte of IP header 430 */ 431 NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1); 432 NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4); 433 NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4); 434 NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16); 435 NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16); 436 NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2); 437 NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2); 438 NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2); 439 NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2); 440 NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2); 441 NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2); 442 NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2); 443 NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2); 444 NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2); 445 NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); 446 NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); 447 NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); 448 NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start, 6); 449 /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ 450 NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2); 451 } 452 453 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) 454 { 455 struct npc_mcam *mcam = &rvu->hw->mcam; 456 u64 *features = &mcam->rx_features; 457 u64 tcp_udp_sctp; 458 int hdr; 459 460 if (is_npc_intf_tx(intf)) 461 features = &mcam->tx_features; 462 463 for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) { 464 if (npc_check_field(rvu, blkaddr, hdr, intf)) 465 *features |= BIT_ULL(hdr); 466 } 467 468 tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) | 469 BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) | 470 BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP); 471 472 /* for tcp/udp/sctp corresponding layer type should be in the key */ 473 if (*features & tcp_udp_sctp) { 474 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf)) 475 *features &= ~tcp_udp_sctp; 476 else 477 *features |= BIT_ULL(NPC_IPPROTO_TCP) | 478 BIT_ULL(NPC_IPPROTO_UDP) | 479 BIT_ULL(NPC_IPPROTO_SCTP); 480 } 481 482 /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */ 483 if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) { 484 *features |= BIT_ULL(NPC_IPPROTO_AH); 485 *features |= BIT_ULL(NPC_IPPROTO_ICMP); 486 *features |= BIT_ULL(NPC_IPPROTO_ICMP6); 487 } 488 489 /* for ESP, check if corresponding layer type is present in the key */ 490 if (npc_check_field(rvu, blkaddr, NPC_LE, intf)) 491 *features |= BIT_ULL(NPC_IPPROTO_ESP); 492 493 /* for vlan corresponding layer type should be in the key */ 494 if (*features & BIT_ULL(NPC_OUTER_VID)) 495 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) 496 *features &= ~BIT_ULL(NPC_OUTER_VID); 497 498 /* for vlan ethertypes corresponding layer type should be in the key */ 499 if (npc_check_field(rvu, blkaddr, NPC_LB, intf)) 500 *features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) | 501 BIT_ULL(NPC_VLAN_ETYPE_STAG); 502 } 503 504 /* Scan key extraction profile and record how fields of our interest 505 * fill the key structure. Also verify Channel and DMAC exists in 506 * key and not overwritten by other header fields. 507 */ 508 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) 509 { 510 struct npc_mcam *mcam = &rvu->hw->mcam; 511 u8 lid, lt, ld, bitnr; 512 u8 key_nibble = 0; 513 u64 cfg; 514 515 /* Scan and note how parse result is going to be in key. 516 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from 517 * parse result in the key. The enabled nibbles from parse result 518 * will be concatenated in key. 519 */ 520 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 521 cfg &= NPC_PARSE_NIBBLE; 522 for_each_set_bit(bitnr, (unsigned long *)&cfg, 31) { 523 npc_scan_parse_result(mcam, bitnr, key_nibble, intf); 524 key_nibble++; 525 } 526 527 /* Scan and note how layer data is going to be in key */ 528 for (lid = 0; lid < NPC_MAX_LID; lid++) { 529 for (lt = 0; lt < NPC_MAX_LT; lt++) { 530 for (ld = 0; ld < NPC_MAX_LD; ld++) { 531 cfg = rvu_read64(rvu, blkaddr, 532 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 533 (intf, lid, lt, ld)); 534 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 535 continue; 536 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, 537 intf); 538 } 539 } 540 } 541 542 return 0; 543 } 544 545 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) 546 { 547 int err; 548 549 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX); 550 if (err) 551 return err; 552 553 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX); 554 if (err) 555 return err; 556 557 /* Channel is mandatory */ 558 if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) { 559 dev_err(rvu->dev, "Channel not present in Key\n"); 560 return -EINVAL; 561 } 562 /* check that none of the fields overwrite channel */ 563 if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) { 564 dev_err(rvu->dev, "Channel cannot be overwritten\n"); 565 return -EINVAL; 566 } 567 /* DMAC should be present in key for unicast filter to work */ 568 if (!npc_is_field_present(rvu, NPC_DMAC, NIX_INTF_RX)) { 569 dev_err(rvu->dev, "DMAC not present in Key\n"); 570 return -EINVAL; 571 } 572 /* check that none of the fields overwrite DMAC */ 573 if (npc_check_overlap(rvu, blkaddr, NPC_DMAC, 0, NIX_INTF_RX)) { 574 dev_err(rvu->dev, "DMAC cannot be overwritten\n"); 575 return -EINVAL; 576 } 577 578 npc_set_features(rvu, blkaddr, NIX_INTF_TX); 579 npc_set_features(rvu, blkaddr, NIX_INTF_RX); 580 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX); 581 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX); 582 583 return 0; 584 } 585 586 int npc_flow_steering_init(struct rvu *rvu, int blkaddr) 587 { 588 struct npc_mcam *mcam = &rvu->hw->mcam; 589 590 INIT_LIST_HEAD(&mcam->mcam_rules); 591 592 return npc_scan_verify_kex(rvu, blkaddr); 593 } 594 595 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) 596 { 597 struct npc_mcam *mcam = &rvu->hw->mcam; 598 u64 *mcam_features = &mcam->rx_features; 599 u64 unsupported; 600 u8 bit; 601 602 if (is_npc_intf_tx(intf)) 603 mcam_features = &mcam->tx_features; 604 605 unsupported = (*mcam_features ^ features) & ~(*mcam_features); 606 if (unsupported) { 607 dev_info(rvu->dev, "Unsupported flow(s):\n"); 608 for_each_set_bit(bit, (unsigned long *)&unsupported, 64) 609 dev_info(rvu->dev, "%s ", npc_get_field_name(bit)); 610 return -EOPNOTSUPP; 611 } 612 613 return 0; 614 } 615 616 /* npc_update_entry - Based on the masks generated during 617 * the key scanning, updates the given entry with value and 618 * masks for the field of interest. Maximum 16 bytes of a packet 619 * header can be extracted by HW hence lo and hi are sufficient. 620 * When field bytes are less than or equal to 8 then hi should be 621 * 0 for value and mask. 622 * 623 * If exact match of value is required then mask should be all 1's. 624 * If any bits in mask are 0 then corresponding bits in value are 625 * dont care. 626 */ 627 static void npc_update_entry(struct rvu *rvu, enum key_fields type, 628 struct mcam_entry *entry, u64 val_lo, 629 u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) 630 { 631 struct npc_mcam *mcam = &rvu->hw->mcam; 632 struct mcam_entry dummy = { {0} }; 633 struct npc_key_field *field; 634 u64 kw1, kw2, kw3; 635 u8 shift; 636 int i; 637 638 field = &mcam->rx_key_fields[type]; 639 if (is_npc_intf_tx(intf)) 640 field = &mcam->tx_key_fields[type]; 641 642 if (!field->nr_kws) 643 return; 644 645 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 646 if (!field->kw_mask[i]) 647 continue; 648 /* place key value in kw[x] */ 649 shift = __ffs64(field->kw_mask[i]); 650 /* update entry value */ 651 kw1 = (val_lo << shift) & field->kw_mask[i]; 652 dummy.kw[i] = kw1; 653 /* update entry mask */ 654 kw1 = (mask_lo << shift) & field->kw_mask[i]; 655 dummy.kw_mask[i] = kw1; 656 657 if (field->nr_kws == 1) 658 break; 659 /* place remaining bits of key value in kw[x + 1] */ 660 if (field->nr_kws == 2) { 661 /* update entry value */ 662 kw2 = shift ? val_lo >> (64 - shift) : 0; 663 kw2 |= (val_hi << shift); 664 kw2 &= field->kw_mask[i + 1]; 665 dummy.kw[i + 1] = kw2; 666 /* update entry mask */ 667 kw2 = shift ? mask_lo >> (64 - shift) : 0; 668 kw2 |= (mask_hi << shift); 669 kw2 &= field->kw_mask[i + 1]; 670 dummy.kw_mask[i + 1] = kw2; 671 break; 672 } 673 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ 674 if (field->nr_kws == 3) { 675 /* update entry value */ 676 kw2 = shift ? val_lo >> (64 - shift) : 0; 677 kw2 |= (val_hi << shift); 678 kw2 &= field->kw_mask[i + 1]; 679 kw3 = shift ? val_hi >> (64 - shift) : 0; 680 kw3 &= field->kw_mask[i + 2]; 681 dummy.kw[i + 1] = kw2; 682 dummy.kw[i + 2] = kw3; 683 /* update entry mask */ 684 kw2 = shift ? mask_lo >> (64 - shift) : 0; 685 kw2 |= (mask_hi << shift); 686 kw2 &= field->kw_mask[i + 1]; 687 kw3 = shift ? mask_hi >> (64 - shift) : 0; 688 kw3 &= field->kw_mask[i + 2]; 689 dummy.kw_mask[i + 1] = kw2; 690 dummy.kw_mask[i + 2] = kw3; 691 break; 692 } 693 } 694 /* dummy is ready with values and masks for given key 695 * field now clear and update input entry with those 696 */ 697 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 698 if (!field->kw_mask[i]) 699 continue; 700 entry->kw[i] &= ~field->kw_mask[i]; 701 entry->kw_mask[i] &= ~field->kw_mask[i]; 702 703 entry->kw[i] |= dummy.kw[i]; 704 entry->kw_mask[i] |= dummy.kw_mask[i]; 705 } 706 } 707 708 #define IPV6_WORDS 4 709 710 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, 711 u64 features, struct flow_msg *pkt, 712 struct flow_msg *mask, 713 struct rvu_npc_mcam_rule *output, u8 intf) 714 { 715 u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS]; 716 u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS]; 717 struct flow_msg *opkt = &output->packet; 718 struct flow_msg *omask = &output->mask; 719 u64 mask_lo, mask_hi; 720 u64 val_lo, val_hi; 721 722 /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet 723 * values to be programmed in MCAM should as below: 724 * val_high: 0xfe80000000000000 725 * val_low: 0x2c6863fffe5e2d0a 726 */ 727 if (features & BIT_ULL(NPC_SIP_IPV6)) { 728 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS); 729 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); 730 731 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1]; 732 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3]; 733 val_hi = (u64)src_ip[0] << 32 | src_ip[1]; 734 val_lo = (u64)src_ip[2] << 32 | src_ip[3]; 735 736 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, 737 mask_lo, mask_hi, intf); 738 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); 739 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); 740 } 741 if (features & BIT_ULL(NPC_DIP_IPV6)) { 742 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS); 743 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); 744 745 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1]; 746 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3]; 747 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; 748 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; 749 750 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, 751 mask_lo, mask_hi, intf); 752 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); 753 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); 754 } 755 } 756 757 static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, 758 u64 features, u8 intf) 759 { 760 bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG)); 761 bool stag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_STAG)); 762 bool vid = !!(features & BIT_ULL(NPC_OUTER_VID)); 763 764 /* If only VLAN id is given then always match outer VLAN id */ 765 if (vid && !ctag && !stag) { 766 npc_update_entry(rvu, NPC_LB, entry, 767 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, 768 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); 769 return; 770 } 771 if (ctag) 772 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0, 773 ~0ULL, 0, intf); 774 if (stag) 775 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0, 776 ~0ULL, 0, intf); 777 } 778 779 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, 780 u64 features, struct flow_msg *pkt, 781 struct flow_msg *mask, 782 struct rvu_npc_mcam_rule *output, u8 intf) 783 { 784 u64 dmac_mask = ether_addr_to_u64(mask->dmac); 785 u64 smac_mask = ether_addr_to_u64(mask->smac); 786 u64 dmac_val = ether_addr_to_u64(pkt->dmac); 787 u64 smac_val = ether_addr_to_u64(pkt->smac); 788 struct flow_msg *opkt = &output->packet; 789 struct flow_msg *omask = &output->mask; 790 791 if (!features) 792 return; 793 794 /* For tcp/udp/sctp LTYPE should be present in entry */ 795 if (features & BIT_ULL(NPC_IPPROTO_TCP)) 796 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, 797 0, ~0ULL, 0, intf); 798 if (features & BIT_ULL(NPC_IPPROTO_UDP)) 799 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, 800 0, ~0ULL, 0, intf); 801 if (features & BIT_ULL(NPC_IPPROTO_SCTP)) 802 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, 803 0, ~0ULL, 0, intf); 804 if (features & BIT_ULL(NPC_IPPROTO_ICMP)) 805 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, 806 0, ~0ULL, 0, intf); 807 if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) 808 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, 809 0, ~0ULL, 0, intf); 810 811 /* For AH, LTYPE should be present in entry */ 812 if (features & BIT_ULL(NPC_IPPROTO_AH)) 813 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, 814 0, ~0ULL, 0, intf); 815 /* For ESP, LTYPE should be present in entry */ 816 if (features & BIT_ULL(NPC_IPPROTO_ESP)) 817 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, 818 0, ~0ULL, 0, intf); 819 820 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ 821 do { \ 822 if (features & BIT_ULL((field))) { \ 823 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ 824 (mask_lo), (mask_hi), intf); \ 825 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ 826 memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ 827 } \ 828 } while (0) 829 830 NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0); 831 NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0); 832 NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0, 833 ntohs(mask->etype), 0); 834 NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0); 835 NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0, 836 ntohl(mask->ip4src), 0); 837 NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0, 838 ntohl(mask->ip4dst), 0); 839 NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0, 840 ntohs(mask->sport), 0); 841 NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0, 842 ntohs(mask->sport), 0); 843 NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0, 844 ntohs(mask->dport), 0); 845 NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0, 846 ntohs(mask->dport), 0); 847 NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0, 848 ntohs(mask->sport), 0); 849 NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0, 850 ntohs(mask->dport), 0); 851 852 NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0, 853 ntohs(mask->vlan_tci), 0); 854 855 npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); 856 npc_update_vlan_features(rvu, entry, features, intf); 857 } 858 859 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, 860 u16 entry) 861 { 862 struct rvu_npc_mcam_rule *iter; 863 864 mutex_lock(&mcam->lock); 865 list_for_each_entry(iter, &mcam->mcam_rules, list) { 866 if (iter->entry == entry) { 867 mutex_unlock(&mcam->lock); 868 return iter; 869 } 870 } 871 mutex_unlock(&mcam->lock); 872 873 return NULL; 874 } 875 876 static void rvu_mcam_add_rule(struct npc_mcam *mcam, 877 struct rvu_npc_mcam_rule *rule) 878 { 879 struct list_head *head = &mcam->mcam_rules; 880 struct rvu_npc_mcam_rule *iter; 881 882 mutex_lock(&mcam->lock); 883 list_for_each_entry(iter, &mcam->mcam_rules, list) { 884 if (iter->entry > rule->entry) 885 break; 886 head = &iter->list; 887 } 888 889 list_add(&rule->list, head); 890 mutex_unlock(&mcam->lock); 891 } 892 893 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 894 struct rvu_npc_mcam_rule *rule) 895 { 896 struct npc_mcam_oper_counter_req free_req = { 0 }; 897 struct msg_rsp free_rsp; 898 899 if (!rule->has_cntr) 900 return; 901 902 free_req.hdr.pcifunc = pcifunc; 903 free_req.cntr = rule->cntr; 904 905 rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp); 906 rule->has_cntr = false; 907 } 908 909 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 910 struct rvu_npc_mcam_rule *rule, 911 struct npc_install_flow_rsp *rsp) 912 { 913 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 914 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 915 int err; 916 917 cntr_req.hdr.pcifunc = pcifunc; 918 cntr_req.contig = true; 919 cntr_req.count = 1; 920 921 /* we try to allocate a counter to track the stats of this 922 * rule. If counter could not be allocated then proceed 923 * without counter because counters are limited than entries. 924 */ 925 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, 926 &cntr_rsp); 927 if (!err && cntr_rsp.count) { 928 rule->cntr = cntr_rsp.cntr; 929 rule->has_cntr = true; 930 rsp->counter = rule->cntr; 931 } else { 932 rsp->counter = err; 933 } 934 } 935 936 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 937 struct mcam_entry *entry, 938 struct npc_install_flow_req *req, 939 u16 target, bool pf_set_vfs_mac) 940 { 941 struct rvu_switch *rswitch = &rvu->rswitch; 942 struct nix_rx_action action; 943 944 if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) 945 req->chan_mask = 0x0; /* Do not care channel */ 946 947 npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, 948 0, NIX_INTF_RX); 949 950 *(u64 *)&action = 0x00; 951 action.pf_func = target; 952 action.op = req->op; 953 action.index = req->index; 954 action.match_id = req->match_id; 955 action.flow_key_alg = req->flow_key_alg; 956 957 if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule) 958 action = pfvf->def_ucast_rule->rx_action; 959 960 entry->action = *(u64 *)&action; 961 962 /* VTAG0 starts at 0th byte of LID_B. 963 * VTAG1 starts at 4th byte of LID_B. 964 */ 965 entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | 966 FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | 967 FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | 968 FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | 969 FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) | 970 FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) | 971 FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) | 972 FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4); 973 } 974 975 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 976 struct mcam_entry *entry, 977 struct npc_install_flow_req *req, u16 target) 978 { 979 struct nix_tx_action action; 980 u64 mask = ~0ULL; 981 982 /* If AF is installing then do not care about 983 * PF_FUNC in Send Descriptor 984 */ 985 if (is_pffunc_af(req->hdr.pcifunc)) 986 mask = 0; 987 988 npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), 989 0, mask, 0, NIX_INTF_TX); 990 991 *(u64 *)&action = 0x00; 992 action.op = req->op; 993 action.index = req->index; 994 action.match_id = req->match_id; 995 996 entry->action = *(u64 *)&action; 997 998 /* VTAG0 starts at 0th byte of LID_B. 999 * VTAG1 starts at 4th byte of LID_B. 1000 */ 1001 entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | 1002 FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | 1003 FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | 1004 FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | 1005 FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) | 1006 FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) | 1007 FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) | 1008 FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24); 1009 } 1010 1011 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, 1012 int nixlf, struct rvu_pfvf *pfvf, 1013 struct npc_install_flow_req *req, 1014 struct npc_install_flow_rsp *rsp, bool enable, 1015 bool pf_set_vfs_mac) 1016 { 1017 struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; 1018 u64 features, installed_features, missing_features = 0; 1019 struct npc_mcam_write_entry_req write_req = { 0 }; 1020 struct npc_mcam *mcam = &rvu->hw->mcam; 1021 struct rvu_npc_mcam_rule dummy = { 0 }; 1022 struct rvu_npc_mcam_rule *rule; 1023 u16 owner = req->hdr.pcifunc; 1024 struct msg_rsp write_rsp; 1025 struct mcam_entry *entry; 1026 int entry_index, err; 1027 bool new = false; 1028 1029 installed_features = req->features; 1030 features = req->features; 1031 entry = &write_req.entry_data; 1032 entry_index = req->entry; 1033 1034 npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, 1035 req->intf); 1036 1037 if (is_npc_intf_rx(req->intf)) 1038 npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); 1039 else 1040 npc_update_tx_entry(rvu, pfvf, entry, req, target); 1041 1042 /* Default unicast rules do not exist for TX */ 1043 if (is_npc_intf_tx(req->intf)) 1044 goto find_rule; 1045 1046 if (req->default_rule) { 1047 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf, 1048 NIXLF_UCAST_ENTRY); 1049 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index); 1050 } 1051 1052 /* update mcam entry with default unicast rule attributes */ 1053 if (def_ucast_rule && (req->default_rule && req->append)) { 1054 missing_features = (def_ucast_rule->features ^ features) & 1055 def_ucast_rule->features; 1056 if (missing_features) 1057 npc_update_flow(rvu, entry, missing_features, 1058 &def_ucast_rule->packet, 1059 &def_ucast_rule->mask, 1060 &dummy, req->intf); 1061 installed_features = req->features | missing_features; 1062 } 1063 1064 find_rule: 1065 rule = rvu_mcam_find_rule(mcam, entry_index); 1066 if (!rule) { 1067 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1068 if (!rule) 1069 return -ENOMEM; 1070 new = true; 1071 } 1072 1073 /* allocate new counter if rule has no counter */ 1074 if (!req->default_rule && req->set_cntr && !rule->has_cntr) 1075 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); 1076 1077 /* if user wants to delete an existing counter for a rule then 1078 * free the counter 1079 */ 1080 if (!req->set_cntr && rule->has_cntr) 1081 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1082 1083 write_req.hdr.pcifunc = owner; 1084 1085 /* AF owns the default rules so change the owner just to relax 1086 * the checks in rvu_mbox_handler_npc_mcam_write_entry 1087 */ 1088 if (req->default_rule) 1089 write_req.hdr.pcifunc = 0; 1090 1091 write_req.entry = entry_index; 1092 write_req.intf = req->intf; 1093 write_req.enable_entry = (u8)enable; 1094 /* if counter is available then clear and use it */ 1095 if (req->set_cntr && rule->has_cntr) { 1096 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00); 1097 write_req.set_cntr = 1; 1098 write_req.cntr = rule->cntr; 1099 } 1100 1101 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, 1102 &write_rsp); 1103 if (err) { 1104 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1105 if (new) 1106 kfree(rule); 1107 return err; 1108 } 1109 /* update rule */ 1110 memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); 1111 memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); 1112 rule->entry = entry_index; 1113 memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); 1114 if (is_npc_intf_tx(req->intf)) 1115 memcpy(&rule->tx_action, &entry->action, 1116 sizeof(struct nix_tx_action)); 1117 rule->vtag_action = entry->vtag_action; 1118 rule->features = installed_features; 1119 rule->default_rule = req->default_rule; 1120 rule->owner = owner; 1121 rule->enable = enable; 1122 rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 1123 rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; 1124 rule->chan &= rule->chan_mask; 1125 if (is_npc_intf_tx(req->intf)) 1126 rule->intf = pfvf->nix_tx_intf; 1127 else 1128 rule->intf = pfvf->nix_rx_intf; 1129 1130 if (new) 1131 rvu_mcam_add_rule(mcam, rule); 1132 if (req->default_rule) 1133 pfvf->def_ucast_rule = rule; 1134 1135 /* VF's MAC address is being changed via PF */ 1136 if (pf_set_vfs_mac) { 1137 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1138 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1139 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1140 } 1141 1142 if (test_bit(PF_SET_VF_CFG, &pfvf->flags) && 1143 req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) 1144 rule->vfvlan_cfg = true; 1145 1146 if (is_npc_intf_rx(req->intf) && req->match_id && 1147 (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS)) 1148 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc, 1149 req->index, req->match_id); 1150 1151 return 0; 1152 } 1153 1154 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, 1155 struct npc_install_flow_req *req, 1156 struct npc_install_flow_rsp *rsp) 1157 { 1158 bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK); 1159 struct rvu_switch *rswitch = &rvu->rswitch; 1160 int blkaddr, nixlf, err; 1161 struct rvu_pfvf *pfvf; 1162 bool pf_set_vfs_mac = false; 1163 bool enable = true; 1164 u16 target; 1165 1166 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1167 if (blkaddr < 0) { 1168 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1169 return NPC_MCAM_INVALID_REQ; 1170 } 1171 1172 if (!is_npc_interface_valid(rvu, req->intf)) 1173 return NPC_FLOW_INTF_INVALID; 1174 1175 if (from_vf && req->default_rule) 1176 return NPC_FLOW_VF_PERM_DENIED; 1177 1178 /* Each PF/VF info is maintained in struct rvu_pfvf. 1179 * rvu_pfvf for the target PF/VF needs to be retrieved 1180 * hence modify pcifunc accordingly. 1181 */ 1182 1183 /* AF installing for a PF/VF */ 1184 if (!req->hdr.pcifunc) 1185 target = req->vf; 1186 /* PF installing for its VF */ 1187 else if (!from_vf && req->vf) { 1188 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf; 1189 pf_set_vfs_mac = req->default_rule && 1190 (req->features & BIT_ULL(NPC_DMAC)); 1191 } 1192 /* msg received from PF/VF */ 1193 else 1194 target = req->hdr.pcifunc; 1195 1196 /* ignore chan_mask in case pf func is not AF, revisit later */ 1197 if (!is_pffunc_af(req->hdr.pcifunc)) 1198 req->chan_mask = 0xFFF; 1199 1200 err = npc_check_unsupported_flows(rvu, req->features, req->intf); 1201 if (err) 1202 return NPC_FLOW_NOT_SUPPORTED; 1203 1204 pfvf = rvu_get_pfvf(rvu, target); 1205 1206 /* PF installing for its VF */ 1207 if (req->hdr.pcifunc && !from_vf && req->vf) 1208 set_bit(PF_SET_VF_CFG, &pfvf->flags); 1209 1210 /* update req destination mac addr */ 1211 if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) && 1212 is_zero_ether_addr(req->packet.dmac)) { 1213 ether_addr_copy(req->packet.dmac, pfvf->mac_addr); 1214 eth_broadcast_addr((u8 *)&req->mask.dmac); 1215 } 1216 1217 /* Proceed if NIXLF is attached or not for TX rules */ 1218 err = nix_get_nixlf(rvu, target, &nixlf, NULL); 1219 if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) 1220 return NPC_FLOW_NO_NIXLF; 1221 1222 /* don't enable rule when nixlf not attached or initialized */ 1223 if (!(is_nixlf_attached(rvu, target) && 1224 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 1225 enable = false; 1226 1227 /* Packets reaching NPC in Tx path implies that a 1228 * NIXLF is properly setup and transmitting. 1229 * Hence rules can be enabled for Tx. 1230 */ 1231 if (is_npc_intf_tx(req->intf)) 1232 enable = true; 1233 1234 /* Do not allow requests from uninitialized VFs */ 1235 if (from_vf && !enable) 1236 return NPC_FLOW_VF_NOT_INIT; 1237 1238 /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ 1239 if (pf_set_vfs_mac && !enable) { 1240 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1241 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1242 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1243 return 0; 1244 } 1245 1246 mutex_lock(&rswitch->switch_lock); 1247 err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, 1248 req, rsp, enable, pf_set_vfs_mac); 1249 mutex_unlock(&rswitch->switch_lock); 1250 1251 return err; 1252 } 1253 1254 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule, 1255 u16 pcifunc) 1256 { 1257 struct npc_mcam_ena_dis_entry_req dis_req = { 0 }; 1258 struct msg_rsp dis_rsp; 1259 1260 if (rule->default_rule) 1261 return 0; 1262 1263 if (rule->has_cntr) 1264 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); 1265 1266 dis_req.hdr.pcifunc = pcifunc; 1267 dis_req.entry = rule->entry; 1268 1269 list_del(&rule->list); 1270 kfree(rule); 1271 1272 return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp); 1273 } 1274 1275 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, 1276 struct npc_delete_flow_req *req, 1277 struct msg_rsp *rsp) 1278 { 1279 struct npc_mcam *mcam = &rvu->hw->mcam; 1280 struct rvu_npc_mcam_rule *iter, *tmp; 1281 u16 pcifunc = req->hdr.pcifunc; 1282 struct list_head del_list; 1283 1284 INIT_LIST_HEAD(&del_list); 1285 1286 mutex_lock(&mcam->lock); 1287 list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) { 1288 if (iter->owner == pcifunc) { 1289 /* All rules */ 1290 if (req->all) { 1291 list_move_tail(&iter->list, &del_list); 1292 /* Range of rules */ 1293 } else if (req->end && iter->entry >= req->start && 1294 iter->entry <= req->end) { 1295 list_move_tail(&iter->list, &del_list); 1296 /* single rule */ 1297 } else if (req->entry == iter->entry) { 1298 list_move_tail(&iter->list, &del_list); 1299 break; 1300 } 1301 } 1302 } 1303 mutex_unlock(&mcam->lock); 1304 1305 list_for_each_entry_safe(iter, tmp, &del_list, list) { 1306 u16 entry = iter->entry; 1307 1308 /* clear the mcam entry target pcifunc */ 1309 mcam->entry2target_pffunc[entry] = 0x0; 1310 if (npc_delete_flow(rvu, iter, pcifunc)) 1311 dev_err(rvu->dev, "rule deletion failed for entry:%u", 1312 entry); 1313 } 1314 1315 return 0; 1316 } 1317 1318 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, 1319 struct rvu_npc_mcam_rule *rule, 1320 struct rvu_pfvf *pfvf) 1321 { 1322 struct npc_mcam_write_entry_req write_req = { 0 }; 1323 struct mcam_entry *entry = &write_req.entry_data; 1324 struct npc_mcam *mcam = &rvu->hw->mcam; 1325 struct msg_rsp rsp; 1326 u8 intf, enable; 1327 int err; 1328 1329 ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); 1330 1331 npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, 1332 entry, &intf, &enable); 1333 1334 npc_update_entry(rvu, NPC_DMAC, entry, 1335 ether_addr_to_u64(pfvf->mac_addr), 0, 1336 0xffffffffffffull, 0, intf); 1337 1338 write_req.hdr.pcifunc = rule->owner; 1339 write_req.entry = rule->entry; 1340 write_req.intf = pfvf->nix_rx_intf; 1341 1342 mutex_unlock(&mcam->lock); 1343 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); 1344 mutex_lock(&mcam->lock); 1345 1346 return err; 1347 } 1348 1349 void npc_mcam_enable_flows(struct rvu *rvu, u16 target) 1350 { 1351 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target); 1352 struct rvu_npc_mcam_rule *def_ucast_rule; 1353 struct npc_mcam *mcam = &rvu->hw->mcam; 1354 struct rvu_npc_mcam_rule *rule; 1355 int blkaddr, bank, index; 1356 u64 def_action; 1357 1358 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1359 if (blkaddr < 0) 1360 return; 1361 1362 def_ucast_rule = pfvf->def_ucast_rule; 1363 1364 mutex_lock(&mcam->lock); 1365 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1366 if (is_npc_intf_rx(rule->intf) && 1367 rule->rx_action.pf_func == target && !rule->enable) { 1368 if (rule->default_rule) { 1369 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1370 rule->entry, true); 1371 rule->enable = true; 1372 continue; 1373 } 1374 1375 if (rule->vfvlan_cfg) 1376 npc_update_dmac_value(rvu, blkaddr, rule, pfvf); 1377 1378 if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) { 1379 if (!def_ucast_rule) 1380 continue; 1381 /* Use default unicast entry action */ 1382 rule->rx_action = def_ucast_rule->rx_action; 1383 def_action = *(u64 *)&def_ucast_rule->rx_action; 1384 bank = npc_get_bank(mcam, rule->entry); 1385 rvu_write64(rvu, blkaddr, 1386 NPC_AF_MCAMEX_BANKX_ACTION 1387 (rule->entry, bank), def_action); 1388 } 1389 1390 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1391 rule->entry, true); 1392 rule->enable = true; 1393 } 1394 } 1395 1396 /* Enable MCAM entries installed by PF with target as VF pcifunc */ 1397 for (index = 0; index < mcam->bmap_entries; index++) { 1398 if (mcam->entry2target_pffunc[index] == target) 1399 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1400 index, true); 1401 } 1402 mutex_unlock(&mcam->lock); 1403 } 1404 1405 void npc_mcam_disable_flows(struct rvu *rvu, u16 target) 1406 { 1407 struct npc_mcam *mcam = &rvu->hw->mcam; 1408 int blkaddr, index; 1409 1410 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1411 if (blkaddr < 0) 1412 return; 1413 1414 mutex_lock(&mcam->lock); 1415 /* Disable MCAM entries installed by PF with target as VF pcifunc */ 1416 for (index = 0; index < mcam->bmap_entries; index++) { 1417 if (mcam->entry2target_pffunc[index] == target) 1418 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1419 index, false); 1420 } 1421 mutex_unlock(&mcam->lock); 1422 } 1423