1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 */ 6 7 #include <linux/bitfield.h> 8 9 #include "rvu_struct.h" 10 #include "rvu_reg.h" 11 #include "rvu.h" 12 #include "npc.h" 13 #include "rvu_npc_fs.h" 14 #include "rvu_npc_hash.h" 15 16 #define NPC_BYTESM GENMASK_ULL(19, 16) 17 #define NPC_HDR_OFFSET GENMASK_ULL(15, 8) 18 #define NPC_KEY_OFFSET GENMASK_ULL(5, 0) 19 #define NPC_LDATA_EN BIT_ULL(7) 20 21 static const char * const npc_flow_names[] = { 22 [NPC_DMAC] = "dmac", 23 [NPC_SMAC] = "smac", 24 [NPC_ETYPE] = "ether type", 25 [NPC_VLAN_ETYPE_CTAG] = "vlan ether type ctag", 26 [NPC_VLAN_ETYPE_STAG] = "vlan ether type stag", 27 [NPC_OUTER_VID] = "outer vlan id", 28 [NPC_TOS] = "tos", 29 [NPC_SIP_IPV4] = "ipv4 source ip", 30 [NPC_DIP_IPV4] = "ipv4 destination ip", 31 [NPC_SIP_IPV6] = "ipv6 source ip", 32 [NPC_DIP_IPV6] = "ipv6 destination ip", 33 [NPC_IPPROTO_TCP] = "ip proto tcp", 34 [NPC_IPPROTO_UDP] = "ip proto udp", 35 [NPC_IPPROTO_SCTP] = "ip proto sctp", 36 [NPC_IPPROTO_ICMP] = "ip proto icmp", 37 [NPC_IPPROTO_ICMP6] = "ip proto icmp6", 38 [NPC_IPPROTO_AH] = "ip proto AH", 39 [NPC_IPPROTO_ESP] = "ip proto ESP", 40 [NPC_SPORT_TCP] = "tcp source port", 41 [NPC_DPORT_TCP] = "tcp destination port", 42 [NPC_SPORT_UDP] = "udp source port", 43 [NPC_DPORT_UDP] = "udp destination port", 44 [NPC_SPORT_SCTP] = "sctp source port", 45 [NPC_DPORT_SCTP] = "sctp destination port", 46 [NPC_LXMB] = "Mcast/Bcast header ", 47 [NPC_UNKNOWN] = "unknown", 48 }; 49 50 const char *npc_get_field_name(u8 hdr) 51 { 52 if (hdr >= ARRAY_SIZE(npc_flow_names)) 53 return npc_flow_names[NPC_UNKNOWN]; 54 55 return npc_flow_names[hdr]; 56 } 57 58 /* Compute keyword masks and figure out the number of keywords a field 59 * spans in the key. 60 */ 61 static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, 62 u8 nr_bits, int start_kwi, int offset, u8 intf) 63 { 64 struct npc_key_field *field = &mcam->rx_key_fields[type]; 65 u8 bits_in_kw; 66 int max_kwi; 67 68 if (mcam->banks_per_entry == 1) 69 max_kwi = 1; /* NPC_MCAM_KEY_X1 */ 70 else if (mcam->banks_per_entry == 2) 71 max_kwi = 3; /* NPC_MCAM_KEY_X2 */ 72 else 73 max_kwi = 6; /* NPC_MCAM_KEY_X4 */ 74 75 if (is_npc_intf_tx(intf)) 76 field = &mcam->tx_key_fields[type]; 77 78 if (offset + nr_bits <= 64) { 79 /* one KW only */ 80 if (start_kwi > max_kwi) 81 return; 82 field->kw_mask[start_kwi] |= GENMASK_ULL(nr_bits - 1, 0) 83 << offset; 84 field->nr_kws = 1; 85 } else if (offset + nr_bits > 64 && 86 offset + nr_bits <= 128) { 87 /* two KWs */ 88 if (start_kwi + 1 > max_kwi) 89 return; 90 /* first KW mask */ 91 bits_in_kw = 64 - offset; 92 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 93 << offset; 94 /* second KW mask i.e. mask for rest of bits */ 95 bits_in_kw = nr_bits + offset - 64; 96 field->kw_mask[start_kwi + 1] |= GENMASK_ULL(bits_in_kw - 1, 0); 97 field->nr_kws = 2; 98 } else { 99 /* three KWs */ 100 if (start_kwi + 2 > max_kwi) 101 return; 102 /* first KW mask */ 103 bits_in_kw = 64 - offset; 104 field->kw_mask[start_kwi] |= GENMASK_ULL(bits_in_kw - 1, 0) 105 << offset; 106 /* second KW mask */ 107 field->kw_mask[start_kwi + 1] = ~0ULL; 108 /* third KW mask i.e. mask for rest of bits */ 109 bits_in_kw = nr_bits + offset - 128; 110 field->kw_mask[start_kwi + 2] |= GENMASK_ULL(bits_in_kw - 1, 0); 111 field->nr_kws = 3; 112 } 113 } 114 115 /* Helper function to figure out whether field exists in the key */ 116 static bool npc_is_field_present(struct rvu *rvu, enum key_fields type, u8 intf) 117 { 118 struct npc_mcam *mcam = &rvu->hw->mcam; 119 struct npc_key_field *input; 120 121 input = &mcam->rx_key_fields[type]; 122 if (is_npc_intf_tx(intf)) 123 input = &mcam->tx_key_fields[type]; 124 125 return input->nr_kws > 0; 126 } 127 128 static bool npc_is_same(struct npc_key_field *input, 129 struct npc_key_field *field) 130 { 131 return memcmp(&input->layer_mdata, &field->layer_mdata, 132 sizeof(struct npc_layer_mdata)) == 0; 133 } 134 135 static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, 136 u64 cfg, u8 lid, u8 lt, u8 intf) 137 { 138 struct npc_key_field *input = &mcam->rx_key_fields[type]; 139 140 if (is_npc_intf_tx(intf)) 141 input = &mcam->tx_key_fields[type]; 142 143 input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 144 input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); 145 input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; 146 input->layer_mdata.ltype = lt; 147 input->layer_mdata.lid = lid; 148 } 149 150 static bool npc_check_overlap_fields(struct npc_key_field *input1, 151 struct npc_key_field *input2) 152 { 153 int kwi; 154 155 /* Fields with same layer id and different ltypes are mutually 156 * exclusive hence they can be overlapped 157 */ 158 if (input1->layer_mdata.lid == input2->layer_mdata.lid && 159 input1->layer_mdata.ltype != input2->layer_mdata.ltype) 160 return false; 161 162 for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { 163 if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) 164 return true; 165 } 166 167 return false; 168 } 169 170 /* Helper function to check whether given field overlaps with any other fields 171 * in the key. Due to limitations on key size and the key extraction profile in 172 * use higher layers can overwrite lower layer's header fields. Hence overlap 173 * needs to be checked. 174 */ 175 static bool npc_check_overlap(struct rvu *rvu, int blkaddr, 176 enum key_fields type, u8 start_lid, u8 intf) 177 { 178 struct npc_mcam *mcam = &rvu->hw->mcam; 179 struct npc_key_field *dummy, *input; 180 int start_kwi, offset; 181 u8 nr_bits, lid, lt, ld; 182 u64 cfg; 183 184 dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; 185 input = &mcam->rx_key_fields[type]; 186 187 if (is_npc_intf_tx(intf)) { 188 dummy = &mcam->tx_key_fields[NPC_UNKNOWN]; 189 input = &mcam->tx_key_fields[type]; 190 } 191 192 for (lid = start_lid; lid < NPC_MAX_LID; lid++) { 193 for (lt = 0; lt < NPC_MAX_LT; lt++) { 194 for (ld = 0; ld < NPC_MAX_LD; ld++) { 195 cfg = rvu_read64(rvu, blkaddr, 196 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 197 (intf, lid, lt, ld)); 198 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 199 continue; 200 memset(dummy, 0, sizeof(struct npc_key_field)); 201 npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, 202 lid, lt, intf); 203 /* exclude input */ 204 if (npc_is_same(input, dummy)) 205 continue; 206 start_kwi = dummy->layer_mdata.key / 8; 207 offset = (dummy->layer_mdata.key * 8) % 64; 208 nr_bits = dummy->layer_mdata.len * 8; 209 /* form KW masks */ 210 npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, 211 start_kwi, offset, intf); 212 /* check any input field bits falls in any 213 * other field bits. 214 */ 215 if (npc_check_overlap_fields(dummy, input)) 216 return true; 217 } 218 } 219 } 220 221 return false; 222 } 223 224 static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, 225 u8 intf) 226 { 227 if (!npc_is_field_present(rvu, type, intf) || 228 npc_check_overlap(rvu, blkaddr, type, 0, intf)) 229 return false; 230 return true; 231 } 232 233 static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, 234 u8 key_nibble, u8 intf) 235 { 236 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 237 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 238 u8 nr_bits = 4; /* bits in a nibble */ 239 u8 type; 240 241 switch (bit_number) { 242 case 40 ... 43: 243 type = NPC_EXACT_RESULT; 244 break; 245 246 default: 247 return; 248 } 249 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 250 } 251 252 static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, 253 u8 key_nibble, u8 intf) 254 { 255 u8 offset = (key_nibble * 4) % 64; /* offset within key word */ 256 u8 kwi = (key_nibble * 4) / 64; /* which word in key */ 257 u8 nr_bits = 4; /* bits in a nibble */ 258 u8 type; 259 260 switch (bit_number) { 261 case 0 ... 2: 262 type = NPC_CHAN; 263 break; 264 case 3: 265 type = NPC_ERRLEV; 266 break; 267 case 4 ... 5: 268 type = NPC_ERRCODE; 269 break; 270 case 6: 271 type = NPC_LXMB; 272 break; 273 /* check for LTYPE only as of now */ 274 case 9: 275 type = NPC_LA; 276 break; 277 case 12: 278 type = NPC_LB; 279 break; 280 case 15: 281 type = NPC_LC; 282 break; 283 case 18: 284 type = NPC_LD; 285 break; 286 case 21: 287 type = NPC_LE; 288 break; 289 case 24: 290 type = NPC_LF; 291 break; 292 case 27: 293 type = NPC_LG; 294 break; 295 case 30: 296 type = NPC_LH; 297 break; 298 default: 299 return; 300 } 301 302 npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); 303 } 304 305 static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) 306 { 307 struct npc_mcam *mcam = &rvu->hw->mcam; 308 struct npc_key_field *key_fields; 309 /* Ether type can come from three layers 310 * (ethernet, single tagged, double tagged) 311 */ 312 struct npc_key_field *etype_ether; 313 struct npc_key_field *etype_tag1; 314 struct npc_key_field *etype_tag2; 315 /* Outer VLAN TCI can come from two layers 316 * (single tagged, double tagged) 317 */ 318 struct npc_key_field *vlan_tag1; 319 struct npc_key_field *vlan_tag2; 320 u64 *features; 321 u8 start_lid; 322 int i; 323 324 key_fields = mcam->rx_key_fields; 325 features = &mcam->rx_features; 326 327 if (is_npc_intf_tx(intf)) { 328 key_fields = mcam->tx_key_fields; 329 features = &mcam->tx_features; 330 } 331 332 /* Handle header fields which can come from multiple layers like 333 * etype, outer vlan tci. These fields should have same position in 334 * the key otherwise to install a mcam rule more than one entry is 335 * needed which complicates mcam space management. 336 */ 337 etype_ether = &key_fields[NPC_ETYPE_ETHER]; 338 etype_tag1 = &key_fields[NPC_ETYPE_TAG1]; 339 etype_tag2 = &key_fields[NPC_ETYPE_TAG2]; 340 vlan_tag1 = &key_fields[NPC_VLAN_TAG1]; 341 vlan_tag2 = &key_fields[NPC_VLAN_TAG2]; 342 343 /* if key profile programmed does not extract Ethertype at all */ 344 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) { 345 dev_err(rvu->dev, "mkex: Ethertype is not extracted.\n"); 346 goto vlan_tci; 347 } 348 349 /* if key profile programmed extracts Ethertype from one layer */ 350 if (etype_ether->nr_kws && !etype_tag1->nr_kws && !etype_tag2->nr_kws) 351 key_fields[NPC_ETYPE] = *etype_ether; 352 if (!etype_ether->nr_kws && etype_tag1->nr_kws && !etype_tag2->nr_kws) 353 key_fields[NPC_ETYPE] = *etype_tag1; 354 if (!etype_ether->nr_kws && !etype_tag1->nr_kws && etype_tag2->nr_kws) 355 key_fields[NPC_ETYPE] = *etype_tag2; 356 357 /* if key profile programmed extracts Ethertype from multiple layers */ 358 if (etype_ether->nr_kws && etype_tag1->nr_kws) { 359 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 360 if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) { 361 dev_err(rvu->dev, "mkex: Etype pos is different for untagged and tagged pkts.\n"); 362 goto vlan_tci; 363 } 364 } 365 key_fields[NPC_ETYPE] = *etype_tag1; 366 } 367 if (etype_ether->nr_kws && etype_tag2->nr_kws) { 368 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 369 if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) { 370 dev_err(rvu->dev, "mkex: Etype pos is different for untagged and double tagged pkts.\n"); 371 goto vlan_tci; 372 } 373 } 374 key_fields[NPC_ETYPE] = *etype_tag2; 375 } 376 if (etype_tag1->nr_kws && etype_tag2->nr_kws) { 377 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 378 if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) { 379 dev_err(rvu->dev, "mkex: Etype pos is different for tagged and double tagged pkts.\n"); 380 goto vlan_tci; 381 } 382 } 383 key_fields[NPC_ETYPE] = *etype_tag2; 384 } 385 386 /* check none of higher layers overwrite Ethertype */ 387 start_lid = key_fields[NPC_ETYPE].layer_mdata.lid + 1; 388 if (npc_check_overlap(rvu, blkaddr, NPC_ETYPE, start_lid, intf)) { 389 dev_err(rvu->dev, "mkex: Ethertype is overwritten by higher layers.\n"); 390 goto vlan_tci; 391 } 392 *features |= BIT_ULL(NPC_ETYPE); 393 vlan_tci: 394 /* if key profile does not extract outer vlan tci at all */ 395 if (!vlan_tag1->nr_kws && !vlan_tag2->nr_kws) { 396 dev_err(rvu->dev, "mkex: Outer vlan tci is not extracted.\n"); 397 goto done; 398 } 399 400 /* if key profile extracts outer vlan tci from one layer */ 401 if (vlan_tag1->nr_kws && !vlan_tag2->nr_kws) 402 key_fields[NPC_OUTER_VID] = *vlan_tag1; 403 if (!vlan_tag1->nr_kws && vlan_tag2->nr_kws) 404 key_fields[NPC_OUTER_VID] = *vlan_tag2; 405 406 /* if key profile extracts outer vlan tci from multiple layers */ 407 if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { 408 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 409 if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) { 410 dev_err(rvu->dev, "mkex: Out vlan tci pos is different for tagged and double tagged pkts.\n"); 411 goto done; 412 } 413 } 414 key_fields[NPC_OUTER_VID] = *vlan_tag2; 415 } 416 /* check none of higher layers overwrite outer vlan tci */ 417 start_lid = key_fields[NPC_OUTER_VID].layer_mdata.lid + 1; 418 if (npc_check_overlap(rvu, blkaddr, NPC_OUTER_VID, start_lid, intf)) { 419 dev_err(rvu->dev, "mkex: Outer vlan tci is overwritten by higher layers.\n"); 420 goto done; 421 } 422 *features |= BIT_ULL(NPC_OUTER_VID); 423 done: 424 return; 425 } 426 427 static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, 428 u8 lt, u64 cfg, u8 intf) 429 { 430 struct npc_mcam *mcam = &rvu->hw->mcam; 431 u8 hdr, key, nr_bytes, bit_offset; 432 u8 la_ltype, la_start; 433 /* starting KW index and starting bit position */ 434 int start_kwi, offset; 435 436 nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; 437 hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); 438 key = FIELD_GET(NPC_KEY_OFFSET, cfg); 439 start_kwi = key / 8; 440 offset = (key * 8) % 64; 441 442 /* For Tx, Layer A has NIX_INST_HDR_S(64 bytes) preceding 443 * ethernet header. 444 */ 445 if (is_npc_intf_tx(intf)) { 446 la_ltype = NPC_LT_LA_IH_NIX_ETHER; 447 la_start = 8; 448 } else { 449 la_ltype = NPC_LT_LA_ETHER; 450 la_start = 0; 451 } 452 453 #define NPC_SCAN_HDR(name, hlid, hlt, hstart, hlen) \ 454 do { \ 455 if (lid == (hlid) && lt == (hlt)) { \ 456 if ((hstart) >= hdr && \ 457 ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ 458 bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ 459 npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ 460 npc_set_kw_masks(mcam, (name), (hlen) * 8, \ 461 start_kwi, offset + bit_offset, intf);\ 462 } \ 463 } \ 464 } while (0) 465 466 /* List LID, LTYPE, start offset from layer and length(in bytes) of 467 * packet header fields below. 468 * Example: Source IP is 4 bytes and starts at 12th byte of IP header 469 */ 470 NPC_SCAN_HDR(NPC_TOS, NPC_LID_LC, NPC_LT_LC_IP, 1, 1); 471 NPC_SCAN_HDR(NPC_SIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 12, 4); 472 NPC_SCAN_HDR(NPC_DIP_IPV4, NPC_LID_LC, NPC_LT_LC_IP, 16, 4); 473 NPC_SCAN_HDR(NPC_SIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 8, 16); 474 NPC_SCAN_HDR(NPC_DIP_IPV6, NPC_LID_LC, NPC_LT_LC_IP6, 24, 16); 475 NPC_SCAN_HDR(NPC_SPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 0, 2); 476 NPC_SCAN_HDR(NPC_DPORT_UDP, NPC_LID_LD, NPC_LT_LD_UDP, 2, 2); 477 NPC_SCAN_HDR(NPC_SPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 0, 2); 478 NPC_SCAN_HDR(NPC_DPORT_TCP, NPC_LID_LD, NPC_LT_LD_TCP, 2, 2); 479 NPC_SCAN_HDR(NPC_SPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 0, 2); 480 NPC_SCAN_HDR(NPC_DPORT_SCTP, NPC_LID_LD, NPC_LT_LD_SCTP, 2, 2); 481 NPC_SCAN_HDR(NPC_ETYPE_ETHER, NPC_LID_LA, NPC_LT_LA_ETHER, 12, 2); 482 NPC_SCAN_HDR(NPC_ETYPE_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 4, 2); 483 NPC_SCAN_HDR(NPC_ETYPE_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 8, 2); 484 NPC_SCAN_HDR(NPC_VLAN_TAG1, NPC_LID_LB, NPC_LT_LB_CTAG, 2, 2); 485 NPC_SCAN_HDR(NPC_VLAN_TAG2, NPC_LID_LB, NPC_LT_LB_STAG_QINQ, 2, 2); 486 NPC_SCAN_HDR(NPC_DMAC, NPC_LID_LA, la_ltype, la_start, 6); 487 /* SMAC follows the DMAC(which is 6 bytes) */ 488 NPC_SCAN_HDR(NPC_SMAC, NPC_LID_LA, la_ltype, la_start + 6, 6); 489 /* PF_FUNC is 2 bytes at 0th byte of NPC_LT_LA_IH_NIX_ETHER */ 490 NPC_SCAN_HDR(NPC_PF_FUNC, NPC_LID_LA, NPC_LT_LA_IH_NIX_ETHER, 0, 2); 491 } 492 493 static void npc_set_features(struct rvu *rvu, int blkaddr, u8 intf) 494 { 495 struct npc_mcam *mcam = &rvu->hw->mcam; 496 u64 *features = &mcam->rx_features; 497 u64 tcp_udp_sctp; 498 int hdr; 499 500 if (is_npc_intf_tx(intf)) 501 features = &mcam->tx_features; 502 503 for (hdr = NPC_DMAC; hdr < NPC_HEADER_FIELDS_MAX; hdr++) { 504 if (npc_check_field(rvu, blkaddr, hdr, intf)) 505 *features |= BIT_ULL(hdr); 506 } 507 508 tcp_udp_sctp = BIT_ULL(NPC_SPORT_TCP) | BIT_ULL(NPC_SPORT_UDP) | 509 BIT_ULL(NPC_DPORT_TCP) | BIT_ULL(NPC_DPORT_UDP) | 510 BIT_ULL(NPC_SPORT_SCTP) | BIT_ULL(NPC_DPORT_SCTP); 511 512 /* for tcp/udp/sctp corresponding layer type should be in the key */ 513 if (*features & tcp_udp_sctp) { 514 if (!npc_check_field(rvu, blkaddr, NPC_LD, intf)) 515 *features &= ~tcp_udp_sctp; 516 else 517 *features |= BIT_ULL(NPC_IPPROTO_TCP) | 518 BIT_ULL(NPC_IPPROTO_UDP) | 519 BIT_ULL(NPC_IPPROTO_SCTP); 520 } 521 522 /* for AH/ICMP/ICMPv6/, check if corresponding layer type is present in the key */ 523 if (npc_check_field(rvu, blkaddr, NPC_LD, intf)) { 524 *features |= BIT_ULL(NPC_IPPROTO_AH); 525 *features |= BIT_ULL(NPC_IPPROTO_ICMP); 526 *features |= BIT_ULL(NPC_IPPROTO_ICMP6); 527 } 528 529 /* for ESP, check if corresponding layer type is present in the key */ 530 if (npc_check_field(rvu, blkaddr, NPC_LE, intf)) 531 *features |= BIT_ULL(NPC_IPPROTO_ESP); 532 533 /* for vlan corresponding layer type should be in the key */ 534 if (*features & BIT_ULL(NPC_OUTER_VID)) 535 if (!npc_check_field(rvu, blkaddr, NPC_LB, intf)) 536 *features &= ~BIT_ULL(NPC_OUTER_VID); 537 538 /* for vlan ethertypes corresponding layer type should be in the key */ 539 if (npc_check_field(rvu, blkaddr, NPC_LB, intf)) 540 *features |= BIT_ULL(NPC_VLAN_ETYPE_CTAG) | 541 BIT_ULL(NPC_VLAN_ETYPE_STAG); 542 543 /* for L2M/L2B/L3M/L3B, check if the type is present in the key */ 544 if (npc_check_field(rvu, blkaddr, NPC_LXMB, intf)) 545 *features |= BIT_ULL(NPC_LXMB); 546 } 547 548 /* Scan key extraction profile and record how fields of our interest 549 * fill the key structure. Also verify Channel and DMAC exists in 550 * key and not overwritten by other header fields. 551 */ 552 static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) 553 { 554 struct npc_mcam *mcam = &rvu->hw->mcam; 555 u8 lid, lt, ld, bitnr; 556 u64 cfg, masked_cfg; 557 u8 key_nibble = 0; 558 559 /* Scan and note how parse result is going to be in key. 560 * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from 561 * parse result in the key. The enabled nibbles from parse result 562 * will be concatenated in key. 563 */ 564 cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); 565 masked_cfg = cfg & NPC_PARSE_NIBBLE; 566 for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) { 567 npc_scan_parse_result(mcam, bitnr, key_nibble, intf); 568 key_nibble++; 569 } 570 571 /* Ignore exact match bits for mcam entries except the first rule 572 * which is drop on hit. This first rule is configured explitcitly by 573 * exact match code. 574 */ 575 masked_cfg = cfg & NPC_EXACT_NIBBLE; 576 bitnr = NPC_EXACT_NIBBLE_START; 577 for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, 578 NPC_EXACT_NIBBLE_START) { 579 npc_scan_exact_result(mcam, bitnr, key_nibble, intf); 580 key_nibble++; 581 } 582 583 /* Scan and note how layer data is going to be in key */ 584 for (lid = 0; lid < NPC_MAX_LID; lid++) { 585 for (lt = 0; lt < NPC_MAX_LT; lt++) { 586 for (ld = 0; ld < NPC_MAX_LD; ld++) { 587 cfg = rvu_read64(rvu, blkaddr, 588 NPC_AF_INTFX_LIDX_LTX_LDX_CFG 589 (intf, lid, lt, ld)); 590 if (!FIELD_GET(NPC_LDATA_EN, cfg)) 591 continue; 592 npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, 593 intf); 594 } 595 } 596 } 597 598 return 0; 599 } 600 601 static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) 602 { 603 int err; 604 605 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_RX); 606 if (err) 607 return err; 608 609 err = npc_scan_kex(rvu, blkaddr, NIX_INTF_TX); 610 if (err) 611 return err; 612 613 /* Channel is mandatory */ 614 if (!npc_is_field_present(rvu, NPC_CHAN, NIX_INTF_RX)) { 615 dev_err(rvu->dev, "Channel not present in Key\n"); 616 return -EINVAL; 617 } 618 /* check that none of the fields overwrite channel */ 619 if (npc_check_overlap(rvu, blkaddr, NPC_CHAN, 0, NIX_INTF_RX)) { 620 dev_err(rvu->dev, "Channel cannot be overwritten\n"); 621 return -EINVAL; 622 } 623 624 npc_set_features(rvu, blkaddr, NIX_INTF_TX); 625 npc_set_features(rvu, blkaddr, NIX_INTF_RX); 626 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_TX); 627 npc_handle_multi_layer_fields(rvu, blkaddr, NIX_INTF_RX); 628 629 return 0; 630 } 631 632 int npc_flow_steering_init(struct rvu *rvu, int blkaddr) 633 { 634 struct npc_mcam *mcam = &rvu->hw->mcam; 635 636 INIT_LIST_HEAD(&mcam->mcam_rules); 637 638 return npc_scan_verify_kex(rvu, blkaddr); 639 } 640 641 static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) 642 { 643 struct npc_mcam *mcam = &rvu->hw->mcam; 644 u64 *mcam_features = &mcam->rx_features; 645 u64 unsupported; 646 u8 bit; 647 648 if (is_npc_intf_tx(intf)) 649 mcam_features = &mcam->tx_features; 650 651 unsupported = (*mcam_features ^ features) & ~(*mcam_features); 652 if (unsupported) { 653 dev_info(rvu->dev, "Unsupported flow(s):\n"); 654 for_each_set_bit(bit, (unsigned long *)&unsupported, 64) 655 dev_info(rvu->dev, "%s ", npc_get_field_name(bit)); 656 return -EOPNOTSUPP; 657 } 658 659 return 0; 660 } 661 662 /* npc_update_entry - Based on the masks generated during 663 * the key scanning, updates the given entry with value and 664 * masks for the field of interest. Maximum 16 bytes of a packet 665 * header can be extracted by HW hence lo and hi are sufficient. 666 * When field bytes are less than or equal to 8 then hi should be 667 * 0 for value and mask. 668 * 669 * If exact match of value is required then mask should be all 1's. 670 * If any bits in mask are 0 then corresponding bits in value are 671 * dont care. 672 */ 673 void npc_update_entry(struct rvu *rvu, enum key_fields type, 674 struct mcam_entry *entry, u64 val_lo, 675 u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) 676 { 677 struct npc_mcam *mcam = &rvu->hw->mcam; 678 struct mcam_entry dummy = { {0} }; 679 struct npc_key_field *field; 680 u64 kw1, kw2, kw3; 681 u8 shift; 682 int i; 683 684 field = &mcam->rx_key_fields[type]; 685 if (is_npc_intf_tx(intf)) 686 field = &mcam->tx_key_fields[type]; 687 688 if (!field->nr_kws) 689 return; 690 691 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 692 if (!field->kw_mask[i]) 693 continue; 694 /* place key value in kw[x] */ 695 shift = __ffs64(field->kw_mask[i]); 696 /* update entry value */ 697 kw1 = (val_lo << shift) & field->kw_mask[i]; 698 dummy.kw[i] = kw1; 699 /* update entry mask */ 700 kw1 = (mask_lo << shift) & field->kw_mask[i]; 701 dummy.kw_mask[i] = kw1; 702 703 if (field->nr_kws == 1) 704 break; 705 /* place remaining bits of key value in kw[x + 1] */ 706 if (field->nr_kws == 2) { 707 /* update entry value */ 708 kw2 = shift ? val_lo >> (64 - shift) : 0; 709 kw2 |= (val_hi << shift); 710 kw2 &= field->kw_mask[i + 1]; 711 dummy.kw[i + 1] = kw2; 712 /* update entry mask */ 713 kw2 = shift ? mask_lo >> (64 - shift) : 0; 714 kw2 |= (mask_hi << shift); 715 kw2 &= field->kw_mask[i + 1]; 716 dummy.kw_mask[i + 1] = kw2; 717 break; 718 } 719 /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ 720 if (field->nr_kws == 3) { 721 /* update entry value */ 722 kw2 = shift ? val_lo >> (64 - shift) : 0; 723 kw2 |= (val_hi << shift); 724 kw2 &= field->kw_mask[i + 1]; 725 kw3 = shift ? val_hi >> (64 - shift) : 0; 726 kw3 &= field->kw_mask[i + 2]; 727 dummy.kw[i + 1] = kw2; 728 dummy.kw[i + 2] = kw3; 729 /* update entry mask */ 730 kw2 = shift ? mask_lo >> (64 - shift) : 0; 731 kw2 |= (mask_hi << shift); 732 kw2 &= field->kw_mask[i + 1]; 733 kw3 = shift ? mask_hi >> (64 - shift) : 0; 734 kw3 &= field->kw_mask[i + 2]; 735 dummy.kw_mask[i + 1] = kw2; 736 dummy.kw_mask[i + 2] = kw3; 737 break; 738 } 739 } 740 /* dummy is ready with values and masks for given key 741 * field now clear and update input entry with those 742 */ 743 for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { 744 if (!field->kw_mask[i]) 745 continue; 746 entry->kw[i] &= ~field->kw_mask[i]; 747 entry->kw_mask[i] &= ~field->kw_mask[i]; 748 749 entry->kw[i] |= dummy.kw[i]; 750 entry->kw_mask[i] |= dummy.kw_mask[i]; 751 } 752 } 753 754 static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, 755 u64 features, struct flow_msg *pkt, 756 struct flow_msg *mask, 757 struct rvu_npc_mcam_rule *output, u8 intf) 758 { 759 u32 src_ip[IPV6_WORDS], src_ip_mask[IPV6_WORDS]; 760 u32 dst_ip[IPV6_WORDS], dst_ip_mask[IPV6_WORDS]; 761 struct flow_msg *opkt = &output->packet; 762 struct flow_msg *omask = &output->mask; 763 u64 mask_lo, mask_hi; 764 u64 val_lo, val_hi; 765 766 /* For an ipv6 address fe80::2c68:63ff:fe5e:2d0a the packet 767 * values to be programmed in MCAM should as below: 768 * val_high: 0xfe80000000000000 769 * val_low: 0x2c6863fffe5e2d0a 770 */ 771 if (features & BIT_ULL(NPC_SIP_IPV6)) { 772 be32_to_cpu_array(src_ip_mask, mask->ip6src, IPV6_WORDS); 773 be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); 774 775 mask_hi = (u64)src_ip_mask[0] << 32 | src_ip_mask[1]; 776 mask_lo = (u64)src_ip_mask[2] << 32 | src_ip_mask[3]; 777 val_hi = (u64)src_ip[0] << 32 | src_ip[1]; 778 val_lo = (u64)src_ip[2] << 32 | src_ip[3]; 779 780 npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, 781 mask_lo, mask_hi, intf); 782 memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); 783 memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); 784 } 785 if (features & BIT_ULL(NPC_DIP_IPV6)) { 786 be32_to_cpu_array(dst_ip_mask, mask->ip6dst, IPV6_WORDS); 787 be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); 788 789 mask_hi = (u64)dst_ip_mask[0] << 32 | dst_ip_mask[1]; 790 mask_lo = (u64)dst_ip_mask[2] << 32 | dst_ip_mask[3]; 791 val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; 792 val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; 793 794 npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, 795 mask_lo, mask_hi, intf); 796 memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); 797 memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); 798 } 799 } 800 801 static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, 802 u64 features, u8 intf) 803 { 804 bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG)); 805 bool stag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_STAG)); 806 bool vid = !!(features & BIT_ULL(NPC_OUTER_VID)); 807 808 /* If only VLAN id is given then always match outer VLAN id */ 809 if (vid && !ctag && !stag) { 810 npc_update_entry(rvu, NPC_LB, entry, 811 NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, 812 NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); 813 return; 814 } 815 if (ctag) 816 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0, 817 ~0ULL, 0, intf); 818 if (stag) 819 npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0, 820 ~0ULL, 0, intf); 821 } 822 823 static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, 824 u64 features, struct flow_msg *pkt, 825 struct flow_msg *mask, 826 struct rvu_npc_mcam_rule *output, u8 intf, 827 int blkaddr) 828 { 829 u64 dmac_mask = ether_addr_to_u64(mask->dmac); 830 u64 smac_mask = ether_addr_to_u64(mask->smac); 831 u64 dmac_val = ether_addr_to_u64(pkt->dmac); 832 u64 smac_val = ether_addr_to_u64(pkt->smac); 833 struct flow_msg *opkt = &output->packet; 834 struct flow_msg *omask = &output->mask; 835 836 if (!features) 837 return; 838 839 /* For tcp/udp/sctp LTYPE should be present in entry */ 840 if (features & BIT_ULL(NPC_IPPROTO_TCP)) 841 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, 842 0, ~0ULL, 0, intf); 843 if (features & BIT_ULL(NPC_IPPROTO_UDP)) 844 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, 845 0, ~0ULL, 0, intf); 846 if (features & BIT_ULL(NPC_IPPROTO_SCTP)) 847 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, 848 0, ~0ULL, 0, intf); 849 if (features & BIT_ULL(NPC_IPPROTO_ICMP)) 850 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, 851 0, ~0ULL, 0, intf); 852 if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) 853 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, 854 0, ~0ULL, 0, intf); 855 856 /* For AH, LTYPE should be present in entry */ 857 if (features & BIT_ULL(NPC_IPPROTO_AH)) 858 npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, 859 0, ~0ULL, 0, intf); 860 /* For ESP, LTYPE should be present in entry */ 861 if (features & BIT_ULL(NPC_IPPROTO_ESP)) 862 npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, 863 0, ~0ULL, 0, intf); 864 865 if (features & BIT_ULL(NPC_LXMB)) { 866 output->lxmb = is_broadcast_ether_addr(pkt->dmac) ? 2 : 1; 867 npc_update_entry(rvu, NPC_LXMB, entry, output->lxmb, 0, 868 output->lxmb, 0, intf); 869 } 870 #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ 871 do { \ 872 if (features & BIT_ULL((field))) { \ 873 npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ 874 (mask_lo), (mask_hi), intf); \ 875 memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ 876 memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ 877 } \ 878 } while (0) 879 880 NPC_WRITE_FLOW(NPC_DMAC, dmac, dmac_val, 0, dmac_mask, 0); 881 882 NPC_WRITE_FLOW(NPC_SMAC, smac, smac_val, 0, smac_mask, 0); 883 NPC_WRITE_FLOW(NPC_ETYPE, etype, ntohs(pkt->etype), 0, 884 ntohs(mask->etype), 0); 885 NPC_WRITE_FLOW(NPC_TOS, tos, pkt->tos, 0, mask->tos, 0); 886 NPC_WRITE_FLOW(NPC_SIP_IPV4, ip4src, ntohl(pkt->ip4src), 0, 887 ntohl(mask->ip4src), 0); 888 NPC_WRITE_FLOW(NPC_DIP_IPV4, ip4dst, ntohl(pkt->ip4dst), 0, 889 ntohl(mask->ip4dst), 0); 890 NPC_WRITE_FLOW(NPC_SPORT_TCP, sport, ntohs(pkt->sport), 0, 891 ntohs(mask->sport), 0); 892 NPC_WRITE_FLOW(NPC_SPORT_UDP, sport, ntohs(pkt->sport), 0, 893 ntohs(mask->sport), 0); 894 NPC_WRITE_FLOW(NPC_DPORT_TCP, dport, ntohs(pkt->dport), 0, 895 ntohs(mask->dport), 0); 896 NPC_WRITE_FLOW(NPC_DPORT_UDP, dport, ntohs(pkt->dport), 0, 897 ntohs(mask->dport), 0); 898 NPC_WRITE_FLOW(NPC_SPORT_SCTP, sport, ntohs(pkt->sport), 0, 899 ntohs(mask->sport), 0); 900 NPC_WRITE_FLOW(NPC_DPORT_SCTP, dport, ntohs(pkt->dport), 0, 901 ntohs(mask->dport), 0); 902 903 NPC_WRITE_FLOW(NPC_OUTER_VID, vlan_tci, ntohs(pkt->vlan_tci), 0, 904 ntohs(mask->vlan_tci), 0); 905 906 npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); 907 npc_update_vlan_features(rvu, entry, features, intf); 908 909 npc_update_field_hash(rvu, intf, entry, blkaddr, features, 910 pkt, mask, opkt, omask); 911 } 912 913 static struct rvu_npc_mcam_rule *rvu_mcam_find_rule(struct npc_mcam *mcam, u16 entry) 914 { 915 struct rvu_npc_mcam_rule *iter; 916 917 mutex_lock(&mcam->lock); 918 list_for_each_entry(iter, &mcam->mcam_rules, list) { 919 if (iter->entry == entry) { 920 mutex_unlock(&mcam->lock); 921 return iter; 922 } 923 } 924 mutex_unlock(&mcam->lock); 925 926 return NULL; 927 } 928 929 static void rvu_mcam_add_rule(struct npc_mcam *mcam, 930 struct rvu_npc_mcam_rule *rule) 931 { 932 struct list_head *head = &mcam->mcam_rules; 933 struct rvu_npc_mcam_rule *iter; 934 935 mutex_lock(&mcam->lock); 936 list_for_each_entry(iter, &mcam->mcam_rules, list) { 937 if (iter->entry > rule->entry) 938 break; 939 head = &iter->list; 940 } 941 942 list_add(&rule->list, head); 943 mutex_unlock(&mcam->lock); 944 } 945 946 static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, 947 struct rvu_npc_mcam_rule *rule) 948 { 949 struct npc_mcam_oper_counter_req free_req = { 0 }; 950 struct msg_rsp free_rsp; 951 952 if (!rule->has_cntr) 953 return; 954 955 free_req.hdr.pcifunc = pcifunc; 956 free_req.cntr = rule->cntr; 957 958 rvu_mbox_handler_npc_mcam_free_counter(rvu, &free_req, &free_rsp); 959 rule->has_cntr = false; 960 } 961 962 static void rvu_mcam_add_counter_to_rule(struct rvu *rvu, u16 pcifunc, 963 struct rvu_npc_mcam_rule *rule, 964 struct npc_install_flow_rsp *rsp) 965 { 966 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 967 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 968 int err; 969 970 cntr_req.hdr.pcifunc = pcifunc; 971 cntr_req.contig = true; 972 cntr_req.count = 1; 973 974 /* we try to allocate a counter to track the stats of this 975 * rule. If counter could not be allocated then proceed 976 * without counter because counters are limited than entries. 977 */ 978 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, 979 &cntr_rsp); 980 if (!err && cntr_rsp.count) { 981 rule->cntr = cntr_rsp.cntr; 982 rule->has_cntr = true; 983 rsp->counter = rule->cntr; 984 } else { 985 rsp->counter = err; 986 } 987 } 988 989 static void npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 990 struct mcam_entry *entry, 991 struct npc_install_flow_req *req, 992 u16 target, bool pf_set_vfs_mac) 993 { 994 struct rvu_switch *rswitch = &rvu->rswitch; 995 struct nix_rx_action action; 996 997 if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) 998 req->chan_mask = 0x0; /* Do not care channel */ 999 1000 npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, 1001 0, NIX_INTF_RX); 1002 1003 *(u64 *)&action = 0x00; 1004 action.pf_func = target; 1005 action.op = req->op; 1006 action.index = req->index; 1007 action.match_id = req->match_id; 1008 action.flow_key_alg = req->flow_key_alg; 1009 1010 if (req->op == NIX_RX_ACTION_DEFAULT && pfvf->def_ucast_rule) 1011 action = pfvf->def_ucast_rule->rx_action; 1012 1013 entry->action = *(u64 *)&action; 1014 1015 /* VTAG0 starts at 0th byte of LID_B. 1016 * VTAG1 starts at 4th byte of LID_B. 1017 */ 1018 entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | 1019 FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | 1020 FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | 1021 FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | 1022 FIELD_PREP(RX_VTAG1_VALID_BIT, req->vtag1_valid) | 1023 FIELD_PREP(RX_VTAG1_TYPE_MASK, req->vtag1_type) | 1024 FIELD_PREP(RX_VTAG1_LID_MASK, NPC_LID_LB) | 1025 FIELD_PREP(RX_VTAG1_RELPTR_MASK, 4); 1026 } 1027 1028 static void npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, 1029 struct mcam_entry *entry, 1030 struct npc_install_flow_req *req, u16 target) 1031 { 1032 struct nix_tx_action action; 1033 u64 mask = ~0ULL; 1034 1035 /* If AF is installing then do not care about 1036 * PF_FUNC in Send Descriptor 1037 */ 1038 if (is_pffunc_af(req->hdr.pcifunc)) 1039 mask = 0; 1040 1041 npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), 1042 0, mask, 0, NIX_INTF_TX); 1043 1044 *(u64 *)&action = 0x00; 1045 action.op = req->op; 1046 action.index = req->index; 1047 action.match_id = req->match_id; 1048 1049 entry->action = *(u64 *)&action; 1050 1051 /* VTAG0 starts at 0th byte of LID_B. 1052 * VTAG1 starts at 4th byte of LID_B. 1053 */ 1054 entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | 1055 FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | 1056 FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | 1057 FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | 1058 FIELD_PREP(TX_VTAG1_DEF_MASK, req->vtag1_def) | 1059 FIELD_PREP(TX_VTAG1_OP_MASK, req->vtag1_op) | 1060 FIELD_PREP(TX_VTAG1_LID_MASK, NPC_LID_LA) | 1061 FIELD_PREP(TX_VTAG1_RELPTR_MASK, 24); 1062 } 1063 1064 static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, 1065 int nixlf, struct rvu_pfvf *pfvf, 1066 struct npc_install_flow_req *req, 1067 struct npc_install_flow_rsp *rsp, bool enable, 1068 bool pf_set_vfs_mac) 1069 { 1070 struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; 1071 u64 features, installed_features, missing_features = 0; 1072 struct npc_mcam_write_entry_req write_req = { 0 }; 1073 struct npc_mcam *mcam = &rvu->hw->mcam; 1074 struct rvu_npc_mcam_rule dummy = { 0 }; 1075 struct rvu_npc_mcam_rule *rule; 1076 u16 owner = req->hdr.pcifunc; 1077 struct msg_rsp write_rsp; 1078 struct mcam_entry *entry; 1079 bool new = false; 1080 u16 entry_index; 1081 int err; 1082 1083 installed_features = req->features; 1084 features = req->features; 1085 entry = &write_req.entry_data; 1086 entry_index = req->entry; 1087 1088 npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, 1089 req->intf, blkaddr); 1090 1091 if (is_npc_intf_rx(req->intf)) 1092 npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); 1093 else 1094 npc_update_tx_entry(rvu, pfvf, entry, req, target); 1095 1096 /* Default unicast rules do not exist for TX */ 1097 if (is_npc_intf_tx(req->intf)) 1098 goto find_rule; 1099 1100 if (req->default_rule) { 1101 entry_index = npc_get_nixlf_mcam_index(mcam, target, nixlf, 1102 NIXLF_UCAST_ENTRY); 1103 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, entry_index); 1104 } 1105 1106 /* update mcam entry with default unicast rule attributes */ 1107 if (def_ucast_rule && (req->default_rule && req->append)) { 1108 missing_features = (def_ucast_rule->features ^ features) & 1109 def_ucast_rule->features; 1110 if (missing_features) 1111 npc_update_flow(rvu, entry, missing_features, 1112 &def_ucast_rule->packet, 1113 &def_ucast_rule->mask, 1114 &dummy, req->intf, 1115 blkaddr); 1116 installed_features = req->features | missing_features; 1117 } 1118 1119 find_rule: 1120 rule = rvu_mcam_find_rule(mcam, entry_index); 1121 if (!rule) { 1122 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1123 if (!rule) 1124 return -ENOMEM; 1125 new = true; 1126 } 1127 1128 /* allocate new counter if rule has no counter */ 1129 if (!req->default_rule && req->set_cntr && !rule->has_cntr) 1130 rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); 1131 1132 /* if user wants to delete an existing counter for a rule then 1133 * free the counter 1134 */ 1135 if (!req->set_cntr && rule->has_cntr) 1136 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1137 1138 write_req.hdr.pcifunc = owner; 1139 1140 /* AF owns the default rules so change the owner just to relax 1141 * the checks in rvu_mbox_handler_npc_mcam_write_entry 1142 */ 1143 if (req->default_rule) 1144 write_req.hdr.pcifunc = 0; 1145 1146 write_req.entry = entry_index; 1147 write_req.intf = req->intf; 1148 write_req.enable_entry = (u8)enable; 1149 /* if counter is available then clear and use it */ 1150 if (req->set_cntr && rule->has_cntr) { 1151 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), 0x00); 1152 write_req.set_cntr = 1; 1153 write_req.cntr = rule->cntr; 1154 } 1155 1156 /* update rule */ 1157 memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); 1158 memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); 1159 rule->entry = entry_index; 1160 memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); 1161 if (is_npc_intf_tx(req->intf)) 1162 memcpy(&rule->tx_action, &entry->action, 1163 sizeof(struct nix_tx_action)); 1164 rule->vtag_action = entry->vtag_action; 1165 rule->features = installed_features; 1166 rule->default_rule = req->default_rule; 1167 rule->owner = owner; 1168 rule->enable = enable; 1169 rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 1170 rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; 1171 rule->chan &= rule->chan_mask; 1172 rule->lxmb = dummy.lxmb; 1173 if (is_npc_intf_tx(req->intf)) 1174 rule->intf = pfvf->nix_tx_intf; 1175 else 1176 rule->intf = pfvf->nix_rx_intf; 1177 1178 if (new) 1179 rvu_mcam_add_rule(mcam, rule); 1180 if (req->default_rule) 1181 pfvf->def_ucast_rule = rule; 1182 1183 /* write to mcam entry registers */ 1184 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, 1185 &write_rsp); 1186 if (err) { 1187 rvu_mcam_remove_counter_from_rule(rvu, owner, rule); 1188 if (new) { 1189 list_del(&rule->list); 1190 kfree(rule); 1191 } 1192 return err; 1193 } 1194 1195 /* VF's MAC address is being changed via PF */ 1196 if (pf_set_vfs_mac) { 1197 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1198 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1199 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1200 } 1201 1202 if (test_bit(PF_SET_VF_CFG, &pfvf->flags) && 1203 req->vtag0_type == NIX_AF_LFX_RX_VTAG_TYPE7) 1204 rule->vfvlan_cfg = true; 1205 1206 if (is_npc_intf_rx(req->intf) && req->match_id && 1207 (req->op == NIX_RX_ACTIONOP_UCAST || req->op == NIX_RX_ACTIONOP_RSS)) 1208 return rvu_nix_setup_ratelimit_aggr(rvu, req->hdr.pcifunc, 1209 req->index, req->match_id); 1210 1211 return 0; 1212 } 1213 1214 int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, 1215 struct npc_install_flow_req *req, 1216 struct npc_install_flow_rsp *rsp) 1217 { 1218 bool from_vf = !!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK); 1219 struct rvu_switch *rswitch = &rvu->rswitch; 1220 int blkaddr, nixlf, err; 1221 struct rvu_pfvf *pfvf; 1222 bool pf_set_vfs_mac = false; 1223 bool enable = true; 1224 u16 target; 1225 1226 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1227 if (blkaddr < 0) { 1228 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1229 return NPC_MCAM_INVALID_REQ; 1230 } 1231 1232 if (!is_npc_interface_valid(rvu, req->intf)) 1233 return NPC_FLOW_INTF_INVALID; 1234 1235 /* If DMAC is not extracted in MKEX, rules installed by AF 1236 * can rely on L2MB bit set by hardware protocol checker for 1237 * broadcast and multicast addresses. 1238 */ 1239 if (npc_check_field(rvu, blkaddr, NPC_DMAC, req->intf)) 1240 goto process_flow; 1241 1242 if (is_pffunc_af(req->hdr.pcifunc)) { 1243 if (is_unicast_ether_addr(req->packet.dmac)) { 1244 dev_err(rvu->dev, 1245 "%s: mkex profile does not support ucast flow\n", 1246 __func__); 1247 return NPC_FLOW_NOT_SUPPORTED; 1248 } 1249 1250 if (!npc_is_field_present(rvu, NPC_LXMB, req->intf)) { 1251 dev_err(rvu->dev, 1252 "%s: mkex profile does not support bcast/mcast flow", 1253 __func__); 1254 return NPC_FLOW_NOT_SUPPORTED; 1255 } 1256 1257 /* Modify feature to use LXMB instead of DMAC */ 1258 req->features &= ~BIT_ULL(NPC_DMAC); 1259 req->features |= BIT_ULL(NPC_LXMB); 1260 } 1261 1262 process_flow: 1263 if (from_vf && req->default_rule) 1264 return NPC_FLOW_VF_PERM_DENIED; 1265 1266 /* Each PF/VF info is maintained in struct rvu_pfvf. 1267 * rvu_pfvf for the target PF/VF needs to be retrieved 1268 * hence modify pcifunc accordingly. 1269 */ 1270 1271 /* AF installing for a PF/VF */ 1272 if (!req->hdr.pcifunc) 1273 target = req->vf; 1274 /* PF installing for its VF */ 1275 else if (!from_vf && req->vf) { 1276 target = (req->hdr.pcifunc & ~RVU_PFVF_FUNC_MASK) | req->vf; 1277 pf_set_vfs_mac = req->default_rule && 1278 (req->features & BIT_ULL(NPC_DMAC)); 1279 } 1280 /* msg received from PF/VF */ 1281 else 1282 target = req->hdr.pcifunc; 1283 1284 /* ignore chan_mask in case pf func is not AF, revisit later */ 1285 if (!is_pffunc_af(req->hdr.pcifunc)) 1286 req->chan_mask = 0xFFF; 1287 1288 err = npc_check_unsupported_flows(rvu, req->features, req->intf); 1289 if (err) 1290 return NPC_FLOW_NOT_SUPPORTED; 1291 1292 pfvf = rvu_get_pfvf(rvu, target); 1293 1294 /* PF installing for its VF */ 1295 if (req->hdr.pcifunc && !from_vf && req->vf) 1296 set_bit(PF_SET_VF_CFG, &pfvf->flags); 1297 1298 /* update req destination mac addr */ 1299 if ((req->features & BIT_ULL(NPC_DMAC)) && is_npc_intf_rx(req->intf) && 1300 is_zero_ether_addr(req->packet.dmac)) { 1301 ether_addr_copy(req->packet.dmac, pfvf->mac_addr); 1302 eth_broadcast_addr((u8 *)&req->mask.dmac); 1303 } 1304 1305 /* Proceed if NIXLF is attached or not for TX rules */ 1306 err = nix_get_nixlf(rvu, target, &nixlf, NULL); 1307 if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) 1308 return NPC_FLOW_NO_NIXLF; 1309 1310 /* don't enable rule when nixlf not attached or initialized */ 1311 if (!(is_nixlf_attached(rvu, target) && 1312 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 1313 enable = false; 1314 1315 /* Packets reaching NPC in Tx path implies that a 1316 * NIXLF is properly setup and transmitting. 1317 * Hence rules can be enabled for Tx. 1318 */ 1319 if (is_npc_intf_tx(req->intf)) 1320 enable = true; 1321 1322 /* Do not allow requests from uninitialized VFs */ 1323 if (from_vf && !enable) 1324 return NPC_FLOW_VF_NOT_INIT; 1325 1326 /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ 1327 if (pf_set_vfs_mac && !enable) { 1328 ether_addr_copy(pfvf->default_mac, req->packet.dmac); 1329 ether_addr_copy(pfvf->mac_addr, req->packet.dmac); 1330 set_bit(PF_SET_VF_MAC, &pfvf->flags); 1331 return 0; 1332 } 1333 1334 mutex_lock(&rswitch->switch_lock); 1335 err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, 1336 req, rsp, enable, pf_set_vfs_mac); 1337 mutex_unlock(&rswitch->switch_lock); 1338 1339 return err; 1340 } 1341 1342 static int npc_delete_flow(struct rvu *rvu, struct rvu_npc_mcam_rule *rule, 1343 u16 pcifunc) 1344 { 1345 struct npc_mcam_ena_dis_entry_req dis_req = { 0 }; 1346 struct msg_rsp dis_rsp; 1347 1348 if (rule->default_rule) 1349 return 0; 1350 1351 if (rule->has_cntr) 1352 rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); 1353 1354 dis_req.hdr.pcifunc = pcifunc; 1355 dis_req.entry = rule->entry; 1356 1357 list_del(&rule->list); 1358 kfree(rule); 1359 1360 return rvu_mbox_handler_npc_mcam_dis_entry(rvu, &dis_req, &dis_rsp); 1361 } 1362 1363 int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, 1364 struct npc_delete_flow_req *req, 1365 struct msg_rsp *rsp) 1366 { 1367 struct npc_mcam *mcam = &rvu->hw->mcam; 1368 struct rvu_npc_mcam_rule *iter, *tmp; 1369 u16 pcifunc = req->hdr.pcifunc; 1370 struct list_head del_list; 1371 1372 INIT_LIST_HEAD(&del_list); 1373 1374 mutex_lock(&mcam->lock); 1375 list_for_each_entry_safe(iter, tmp, &mcam->mcam_rules, list) { 1376 if (iter->owner == pcifunc) { 1377 /* All rules */ 1378 if (req->all) { 1379 list_move_tail(&iter->list, &del_list); 1380 /* Range of rules */ 1381 } else if (req->end && iter->entry >= req->start && 1382 iter->entry <= req->end) { 1383 list_move_tail(&iter->list, &del_list); 1384 /* single rule */ 1385 } else if (req->entry == iter->entry) { 1386 list_move_tail(&iter->list, &del_list); 1387 break; 1388 } 1389 } 1390 } 1391 mutex_unlock(&mcam->lock); 1392 1393 list_for_each_entry_safe(iter, tmp, &del_list, list) { 1394 u16 entry = iter->entry; 1395 1396 /* clear the mcam entry target pcifunc */ 1397 mcam->entry2target_pffunc[entry] = 0x0; 1398 if (npc_delete_flow(rvu, iter, pcifunc)) 1399 dev_err(rvu->dev, "rule deletion failed for entry:%u", 1400 entry); 1401 } 1402 1403 return 0; 1404 } 1405 1406 static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, 1407 struct rvu_npc_mcam_rule *rule, 1408 struct rvu_pfvf *pfvf) 1409 { 1410 struct npc_mcam_write_entry_req write_req = { 0 }; 1411 struct mcam_entry *entry = &write_req.entry_data; 1412 struct npc_mcam *mcam = &rvu->hw->mcam; 1413 struct msg_rsp rsp; 1414 u8 intf, enable; 1415 int err; 1416 1417 ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); 1418 1419 npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, 1420 entry, &intf, &enable); 1421 1422 npc_update_entry(rvu, NPC_DMAC, entry, 1423 ether_addr_to_u64(pfvf->mac_addr), 0, 1424 0xffffffffffffull, 0, intf); 1425 1426 write_req.hdr.pcifunc = rule->owner; 1427 write_req.entry = rule->entry; 1428 write_req.intf = pfvf->nix_rx_intf; 1429 1430 mutex_unlock(&mcam->lock); 1431 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); 1432 mutex_lock(&mcam->lock); 1433 1434 return err; 1435 } 1436 1437 void npc_mcam_enable_flows(struct rvu *rvu, u16 target) 1438 { 1439 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, target); 1440 struct rvu_npc_mcam_rule *def_ucast_rule; 1441 struct npc_mcam *mcam = &rvu->hw->mcam; 1442 struct rvu_npc_mcam_rule *rule; 1443 int blkaddr, bank, index; 1444 u64 def_action; 1445 1446 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1447 if (blkaddr < 0) 1448 return; 1449 1450 def_ucast_rule = pfvf->def_ucast_rule; 1451 1452 mutex_lock(&mcam->lock); 1453 list_for_each_entry(rule, &mcam->mcam_rules, list) { 1454 if (is_npc_intf_rx(rule->intf) && 1455 rule->rx_action.pf_func == target && !rule->enable) { 1456 if (rule->default_rule) { 1457 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1458 rule->entry, true); 1459 rule->enable = true; 1460 continue; 1461 } 1462 1463 if (rule->vfvlan_cfg) 1464 npc_update_dmac_value(rvu, blkaddr, rule, pfvf); 1465 1466 if (rule->rx_action.op == NIX_RX_ACTION_DEFAULT) { 1467 if (!def_ucast_rule) 1468 continue; 1469 /* Use default unicast entry action */ 1470 rule->rx_action = def_ucast_rule->rx_action; 1471 def_action = *(u64 *)&def_ucast_rule->rx_action; 1472 bank = npc_get_bank(mcam, rule->entry); 1473 rvu_write64(rvu, blkaddr, 1474 NPC_AF_MCAMEX_BANKX_ACTION 1475 (rule->entry, bank), def_action); 1476 } 1477 1478 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1479 rule->entry, true); 1480 rule->enable = true; 1481 } 1482 } 1483 1484 /* Enable MCAM entries installed by PF with target as VF pcifunc */ 1485 for (index = 0; index < mcam->bmap_entries; index++) { 1486 if (mcam->entry2target_pffunc[index] == target) 1487 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1488 index, true); 1489 } 1490 mutex_unlock(&mcam->lock); 1491 } 1492 1493 void npc_mcam_disable_flows(struct rvu *rvu, u16 target) 1494 { 1495 struct npc_mcam *mcam = &rvu->hw->mcam; 1496 int blkaddr, index; 1497 1498 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1499 if (blkaddr < 0) 1500 return; 1501 1502 mutex_lock(&mcam->lock); 1503 /* Disable MCAM entries installed by PF with target as VF pcifunc */ 1504 for (index = 0; index < mcam->bmap_entries; index++) { 1505 if (mcam->entry2target_pffunc[index] == target) 1506 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1507 index, false); 1508 } 1509 mutex_unlock(&mcam->lock); 1510 } 1511 1512 /* single drop on non hit rule starting from 0th index. This an extension 1513 * to RPM mac filter to support more rules. 1514 */ 1515 int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, 1516 u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, 1517 u64 bcast_mcast_val, u64 bcast_mcast_mask) 1518 { 1519 struct npc_mcam_alloc_counter_req cntr_req = { 0 }; 1520 struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; 1521 struct npc_mcam_write_entry_req req = { 0 }; 1522 struct npc_mcam *mcam = &rvu->hw->mcam; 1523 struct rvu_npc_mcam_rule *rule; 1524 struct msg_rsp rsp; 1525 bool enabled; 1526 int blkaddr; 1527 int err; 1528 1529 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1530 if (blkaddr < 0) { 1531 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1532 return -ENODEV; 1533 } 1534 1535 /* Bail out if no exact match support */ 1536 if (!rvu_npc_exact_has_match_table(rvu)) { 1537 dev_info(rvu->dev, "%s: No support for exact match feature\n", __func__); 1538 return -EINVAL; 1539 } 1540 1541 /* If 0th entry is already used, return err */ 1542 enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_idx); 1543 if (enabled) { 1544 dev_err(rvu->dev, "%s: failed to add single drop on non hit rule at %d th index\n", 1545 __func__, mcam_idx); 1546 return -EINVAL; 1547 } 1548 1549 /* Add this entry to mcam rules list */ 1550 rule = kzalloc(sizeof(*rule), GFP_KERNEL); 1551 if (!rule) 1552 return -ENOMEM; 1553 1554 /* Disable rule by default. Enable rule when first dmac filter is 1555 * installed 1556 */ 1557 rule->enable = false; 1558 rule->chan = chan_val; 1559 rule->chan_mask = chan_mask; 1560 rule->entry = mcam_idx; 1561 rvu_mcam_add_rule(mcam, rule); 1562 1563 /* Reserve slot 0 */ 1564 npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx); 1565 1566 /* Allocate counter for this single drop on non hit rule */ 1567 cntr_req.hdr.pcifunc = 0; /* AF request */ 1568 cntr_req.contig = true; 1569 cntr_req.count = 1; 1570 err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 1571 if (err) { 1572 dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n", 1573 __func__, err); 1574 return -EFAULT; 1575 } 1576 *counter_idx = cntr_rsp.cntr; 1577 1578 /* Fill in fields for this mcam entry */ 1579 npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0, 1580 exact_mask, 0, NIX_INTF_RX); 1581 npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0, 1582 chan_mask, 0, NIX_INTF_RX); 1583 npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0, 1584 bcast_mcast_mask, 0, NIX_INTF_RX); 1585 1586 req.intf = NIX_INTF_RX; 1587 req.set_cntr = true; 1588 req.cntr = cntr_rsp.cntr; 1589 req.entry = mcam_idx; 1590 1591 err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp); 1592 if (err) { 1593 dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n", 1594 __func__, mcam_idx); 1595 return err; 1596 } 1597 1598 dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", 1599 __func__, mcam_idx, req.cntr); 1600 1601 /* disable entry at Bank 0, index 0 */ 1602 npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false); 1603 1604 return 0; 1605 } 1606