1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 19 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 20 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 21 22 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 23 #define NPC_HW_TSTAMP_OFFSET 8ULL 24 #define NPC_KEX_CHAN_MASK 0xFFFULL 25 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 26 27 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 28 29 static const char def_pfl_name[] = "default"; 30 31 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 32 int blkaddr, u16 pcifunc); 33 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 34 u16 pcifunc); 35 36 bool is_npc_intf_tx(u8 intf) 37 { 38 return !!(intf & 0x1); 39 } 40 41 bool is_npc_intf_rx(u8 intf) 42 { 43 return !(intf & 0x1); 44 } 45 46 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 47 { 48 struct rvu_hwinfo *hw = rvu->hw; 49 50 return intf < hw->npc_intfs; 51 } 52 53 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 54 { 55 /* Due to a HW issue in these silicon versions, parse nibble enable 56 * configuration has to be identical for both Rx and Tx interfaces. 57 */ 58 if (is_rvu_96xx_B0(rvu)) 59 return nibble_ena; 60 return 0; 61 } 62 63 static int npc_mcam_verify_pf_func(struct rvu *rvu, 64 struct mcam_entry *entry_data, u8 intf, 65 u16 pcifunc) 66 { 67 u16 pf_func, pf_func_mask; 68 69 if (is_npc_intf_rx(intf)) 70 return 0; 71 72 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 73 NPC_KEX_PF_FUNC_MASK; 74 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 75 76 pf_func = be16_to_cpu((__force __be16)pf_func); 77 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 78 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 79 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 80 return -EINVAL; 81 82 return 0; 83 } 84 85 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 86 { 87 int blkaddr; 88 u64 val = 0; 89 90 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 91 if (blkaddr < 0) 92 return; 93 94 /* Config CPI base for the PKIND */ 95 val = pkind | 1ULL << 62; 96 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 97 } 98 99 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 100 { 101 struct npc_pkind *pkind = &rvu->hw->pkind; 102 u32 map; 103 int i; 104 105 for (i = 0; i < pkind->rsrc.max; i++) { 106 map = pkind->pfchan_map[i]; 107 if (((map >> 16) & 0x3F) == pf) 108 return i; 109 } 110 return -1; 111 } 112 113 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 114 115 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 116 { 117 int pkind, blkaddr; 118 u64 val; 119 120 pkind = rvu_npc_get_pkind(rvu, pf); 121 if (pkind < 0) { 122 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 123 return -EINVAL; 124 } 125 126 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 127 if (blkaddr < 0) { 128 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 129 return -EINVAL; 130 } 131 132 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 133 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 134 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 135 if (enable) 136 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 137 NPC_HW_TSTAMP_OFFSET); 138 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 139 140 return 0; 141 } 142 143 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 144 int nixlf) 145 { 146 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 147 struct rvu *rvu = hw->rvu; 148 int blkaddr = 0, max = 0; 149 struct rvu_block *block; 150 struct rvu_pfvf *pfvf; 151 152 pfvf = rvu_get_pfvf(rvu, pcifunc); 153 /* Given a PF/VF and NIX LF number calculate the unicast mcam 154 * entry index based on the NIX block assigned to the PF/VF. 155 */ 156 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 157 while (blkaddr) { 158 if (pfvf->nix_blkaddr == blkaddr) 159 break; 160 block = &rvu->hw->block[blkaddr]; 161 max += block->lf.max; 162 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 163 } 164 165 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 166 } 167 168 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 169 u16 pcifunc, int nixlf, int type) 170 { 171 int pf = rvu_get_pf(pcifunc); 172 int index; 173 174 /* Check if this is for a PF */ 175 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 176 /* Reserved entries exclude PF0 */ 177 pf--; 178 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 179 /* Broadcast address matching entry should be first so 180 * that the packet can be replicated to all VFs. 181 */ 182 if (type == NIXLF_BCAST_ENTRY) 183 return index; 184 else if (type == NIXLF_ALLMULTI_ENTRY) 185 return index + 1; 186 else if (type == NIXLF_PROMISC_ENTRY) 187 return index + 2; 188 } 189 190 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 191 } 192 193 int npc_get_bank(struct npc_mcam *mcam, int index) 194 { 195 int bank = index / mcam->banksize; 196 197 /* 0,1 & 2,3 banks are combined for this keysize */ 198 if (mcam->keysize == NPC_MCAM_KEY_X2) 199 return bank ? 2 : 0; 200 201 return bank; 202 } 203 204 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 205 int blkaddr, int index) 206 { 207 int bank = npc_get_bank(mcam, index); 208 u64 cfg; 209 210 index &= (mcam->banksize - 1); 211 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 212 return (cfg & 1); 213 } 214 215 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 216 int blkaddr, int index, bool enable) 217 { 218 int bank = npc_get_bank(mcam, index); 219 int actbank = bank; 220 221 index &= (mcam->banksize - 1); 222 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 223 rvu_write64(rvu, blkaddr, 224 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 225 enable ? 1 : 0); 226 } 227 } 228 229 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 230 int blkaddr, int index) 231 { 232 int bank = npc_get_bank(mcam, index); 233 int actbank = bank; 234 235 index &= (mcam->banksize - 1); 236 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 237 rvu_write64(rvu, blkaddr, 238 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 239 rvu_write64(rvu, blkaddr, 240 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 241 242 rvu_write64(rvu, blkaddr, 243 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 244 rvu_write64(rvu, blkaddr, 245 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 246 247 rvu_write64(rvu, blkaddr, 248 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 249 rvu_write64(rvu, blkaddr, 250 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 251 } 252 } 253 254 static void npc_get_keyword(struct mcam_entry *entry, int idx, 255 u64 *cam0, u64 *cam1) 256 { 257 u64 kw_mask = 0x00; 258 259 #define CAM_MASK(n) (BIT_ULL(n) - 1) 260 261 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 262 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 263 * 264 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 265 */ 266 switch (idx) { 267 case 0: 268 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 269 *cam1 = entry->kw[0]; 270 kw_mask = entry->kw_mask[0]; 271 break; 272 case 1: 273 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 274 *cam1 = entry->kw[1] & CAM_MASK(48); 275 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 276 break; 277 case 2: 278 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 279 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 280 */ 281 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 282 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 283 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 284 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 285 break; 286 case 3: 287 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 288 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 289 */ 290 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 291 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 292 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 293 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 294 break; 295 case 4: 296 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 297 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 298 */ 299 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 300 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 301 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 302 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 303 break; 304 case 5: 305 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 306 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 307 */ 308 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 309 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 310 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 311 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 312 break; 313 case 6: 314 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 315 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 316 */ 317 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 318 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 319 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 320 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 321 break; 322 case 7: 323 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 324 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 325 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 326 break; 327 } 328 329 *cam1 &= kw_mask; 330 *cam0 = ~*cam1 & kw_mask; 331 } 332 333 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 334 u64 cam0, u64 cam1) 335 { 336 /* Similar to npc_get_keyword, but fills mcam_entry structure from 337 * CAM registers. 338 */ 339 switch (idx) { 340 case 0: 341 entry->kw[0] = cam1; 342 entry->kw_mask[0] = cam1 ^ cam0; 343 break; 344 case 1: 345 entry->kw[1] = cam1; 346 entry->kw_mask[1] = cam1 ^ cam0; 347 break; 348 case 2: 349 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 350 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 351 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 352 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 353 break; 354 case 3: 355 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 356 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 357 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 358 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 359 break; 360 case 4: 361 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 362 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 363 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 364 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 365 break; 366 case 5: 367 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 368 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 369 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 370 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 371 break; 372 case 6: 373 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 374 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 375 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 376 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 377 break; 378 case 7: 379 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 380 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 381 break; 382 } 383 } 384 385 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 386 int blkaddr, u16 pf_func) 387 { 388 int bank, nixlf, index; 389 390 /* get ucast entry rule entry index */ 391 nix_get_nixlf(rvu, pf_func, &nixlf, NULL); 392 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 393 NIXLF_UCAST_ENTRY); 394 bank = npc_get_bank(mcam, index); 395 index &= (mcam->banksize - 1); 396 397 return rvu_read64(rvu, blkaddr, 398 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 399 } 400 401 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 402 int blkaddr, int index, struct mcam_entry *entry, 403 bool *enable) 404 { 405 u16 owner, target_func; 406 struct rvu_pfvf *pfvf; 407 u64 rx_action; 408 409 owner = mcam->entry2pfvf_map[index]; 410 target_func = (entry->action >> 4) & 0xffff; 411 /* do nothing when target is LBK/PF or owner is not PF */ 412 if (is_pffunc_af(owner) || is_afvf(target_func) || 413 (owner & RVU_PFVF_FUNC_MASK) || 414 !(target_func & RVU_PFVF_FUNC_MASK)) 415 return; 416 417 /* save entry2target_pffunc */ 418 pfvf = rvu_get_pfvf(rvu, target_func); 419 mcam->entry2target_pffunc[index] = target_func; 420 421 /* don't enable rule when nixlf not attached or initialized */ 422 if (!(is_nixlf_attached(rvu, target_func) && 423 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 424 *enable = false; 425 426 /* copy VF default entry action to the VF mcam entry */ 427 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 428 target_func); 429 if (rx_action) 430 entry->action = rx_action; 431 } 432 433 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 434 int blkaddr, int index, u8 intf, 435 struct mcam_entry *entry, bool enable) 436 { 437 int bank = npc_get_bank(mcam, index); 438 int kw = 0, actbank, actindex; 439 u8 tx_intf_mask = ~intf & 0x3; 440 u8 tx_intf = intf; 441 u64 cam0, cam1; 442 443 actbank = bank; /* Save bank id, to set action later on */ 444 actindex = index; 445 index &= (mcam->banksize - 1); 446 447 /* Disable before mcam entry update */ 448 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 449 450 /* Clear mcam entry to avoid writes being suppressed by NPC */ 451 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 452 453 /* CAM1 takes the comparison value and 454 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 455 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 456 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 457 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 458 */ 459 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 460 /* Interface should be set in all banks */ 461 if (is_npc_intf_tx(intf)) { 462 /* Last bit must be set and rest don't care 463 * for TX interfaces 464 */ 465 tx_intf_mask = 0x1; 466 tx_intf = intf & tx_intf_mask; 467 tx_intf_mask = ~tx_intf & tx_intf_mask; 468 } 469 470 rvu_write64(rvu, blkaddr, 471 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 472 tx_intf); 473 rvu_write64(rvu, blkaddr, 474 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 475 tx_intf_mask); 476 477 /* Set the match key */ 478 npc_get_keyword(entry, kw, &cam0, &cam1); 479 rvu_write64(rvu, blkaddr, 480 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 481 rvu_write64(rvu, blkaddr, 482 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 483 484 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 485 rvu_write64(rvu, blkaddr, 486 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 487 rvu_write64(rvu, blkaddr, 488 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 489 } 490 491 /* PF installing VF rule */ 492 if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) 493 npc_fixup_vf_rule(rvu, mcam, blkaddr, index, entry, &enable); 494 495 /* Set 'action' */ 496 rvu_write64(rvu, blkaddr, 497 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 498 499 /* Set TAG 'action' */ 500 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 501 entry->vtag_action); 502 503 /* Enable the entry */ 504 if (enable) 505 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 506 } 507 508 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 509 int blkaddr, u16 src, 510 struct mcam_entry *entry, u8 *intf, u8 *ena) 511 { 512 int sbank = npc_get_bank(mcam, src); 513 int bank, kw = 0; 514 u64 cam0, cam1; 515 516 src &= (mcam->banksize - 1); 517 bank = sbank; 518 519 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 520 cam1 = rvu_read64(rvu, blkaddr, 521 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 522 cam0 = rvu_read64(rvu, blkaddr, 523 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 524 npc_fill_entryword(entry, kw, cam0, cam1); 525 526 cam1 = rvu_read64(rvu, blkaddr, 527 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 528 cam0 = rvu_read64(rvu, blkaddr, 529 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 530 npc_fill_entryword(entry, kw + 1, cam0, cam1); 531 } 532 533 entry->action = rvu_read64(rvu, blkaddr, 534 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 535 entry->vtag_action = 536 rvu_read64(rvu, blkaddr, 537 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 538 *intf = rvu_read64(rvu, blkaddr, 539 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 540 *ena = rvu_read64(rvu, blkaddr, 541 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 542 } 543 544 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 545 int blkaddr, u16 src, u16 dest) 546 { 547 int dbank = npc_get_bank(mcam, dest); 548 int sbank = npc_get_bank(mcam, src); 549 u64 cfg, sreg, dreg; 550 int bank, i; 551 552 src &= (mcam->banksize - 1); 553 dest &= (mcam->banksize - 1); 554 555 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 556 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 557 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 558 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 559 for (i = 0; i < 6; i++) { 560 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 561 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 562 } 563 } 564 565 /* Copy action */ 566 cfg = rvu_read64(rvu, blkaddr, 567 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 568 rvu_write64(rvu, blkaddr, 569 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 570 571 /* Copy TAG action */ 572 cfg = rvu_read64(rvu, blkaddr, 573 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 574 rvu_write64(rvu, blkaddr, 575 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 576 577 /* Enable or disable */ 578 cfg = rvu_read64(rvu, blkaddr, 579 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 580 rvu_write64(rvu, blkaddr, 581 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 582 } 583 584 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 585 int blkaddr, int index) 586 { 587 int bank = npc_get_bank(mcam, index); 588 589 index &= (mcam->banksize - 1); 590 return rvu_read64(rvu, blkaddr, 591 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 592 } 593 594 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 595 int nixlf, u64 chan, u8 *mac_addr) 596 { 597 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 598 struct npc_install_flow_req req = { 0 }; 599 struct npc_install_flow_rsp rsp = { 0 }; 600 struct npc_mcam *mcam = &rvu->hw->mcam; 601 struct nix_rx_action action; 602 int blkaddr, index; 603 604 /* AF's and SDP VFs work in promiscuous mode */ 605 if (is_afvf(pcifunc) || is_sdp_vf(pcifunc)) 606 return; 607 608 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 609 if (blkaddr < 0) 610 return; 611 612 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 613 nixlf, NIXLF_UCAST_ENTRY); 614 615 /* Don't change the action if entry is already enabled 616 * Otherwise RSS action may get overwritten. 617 */ 618 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 619 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 620 blkaddr, index); 621 } else { 622 *(u64 *)&action = 0x00; 623 action.op = NIX_RX_ACTIONOP_UCAST; 624 action.pf_func = pcifunc; 625 } 626 627 req.default_rule = 1; 628 ether_addr_copy(req.packet.dmac, mac_addr); 629 eth_broadcast_addr((u8 *)&req.mask.dmac); 630 req.features = BIT_ULL(NPC_DMAC); 631 req.channel = chan; 632 req.chan_mask = 0xFFFU; 633 req.intf = pfvf->nix_rx_intf; 634 req.op = action.op; 635 req.hdr.pcifunc = 0; /* AF is requester */ 636 req.vf = action.pf_func; 637 req.index = action.index; 638 req.match_id = action.match_id; 639 req.flow_key_alg = action.flow_key_alg; 640 641 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 642 } 643 644 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 645 int nixlf, u64 chan, u8 chan_cnt) 646 { 647 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 648 struct npc_install_flow_req req = { 0 }; 649 struct npc_install_flow_rsp rsp = { 0 }; 650 struct npc_mcam *mcam = &rvu->hw->mcam; 651 struct rvu_hwinfo *hw = rvu->hw; 652 int blkaddr, ucast_idx, index; 653 struct nix_rx_action action; 654 u64 relaxed_mask; 655 656 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 657 return; 658 659 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 660 if (blkaddr < 0) 661 return; 662 663 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 664 nixlf, NIXLF_PROMISC_ENTRY); 665 666 if (is_cgx_vf(rvu, pcifunc)) 667 index = npc_get_nixlf_mcam_index(mcam, 668 pcifunc & ~RVU_PFVF_FUNC_MASK, 669 nixlf, NIXLF_PROMISC_ENTRY); 670 671 /* If the corresponding PF's ucast action is RSS, 672 * use the same action for promisc also 673 */ 674 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 675 nixlf, NIXLF_UCAST_ENTRY); 676 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 677 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 678 blkaddr, ucast_idx); 679 680 if (action.op != NIX_RX_ACTIONOP_RSS) { 681 *(u64 *)&action = 0x00; 682 action.op = NIX_RX_ACTIONOP_UCAST; 683 } 684 685 /* RX_ACTION set to MCAST for CGX PF's */ 686 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 687 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { 688 *(u64 *)&action = 0x00; 689 action.op = NIX_RX_ACTIONOP_MCAST; 690 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 691 action.index = pfvf->promisc_mce_idx; 692 } 693 694 /* For cn10k the upper two bits of the channel number are 695 * cpt channel number. with masking out these bits in the 696 * mcam entry, same entry used for NIX will allow packets 697 * received from cpt for parsing. 698 */ 699 if (!is_rvu_otx2(rvu)) { 700 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 701 } else { 702 req.chan_mask = 0xFFFU; 703 } 704 705 if (chan_cnt > 1) { 706 if (!is_power_of_2(chan_cnt)) { 707 dev_err(rvu->dev, 708 "%s: channel count more than 1, must be power of 2\n", __func__); 709 return; 710 } 711 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 712 ilog2(chan_cnt)); 713 req.chan_mask &= relaxed_mask; 714 } 715 716 req.channel = chan; 717 req.intf = pfvf->nix_rx_intf; 718 req.entry = index; 719 req.op = action.op; 720 req.hdr.pcifunc = 0; /* AF is requester */ 721 req.vf = pcifunc; 722 req.index = action.index; 723 req.match_id = action.match_id; 724 req.flow_key_alg = action.flow_key_alg; 725 726 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 727 } 728 729 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 730 int nixlf, bool enable) 731 { 732 struct npc_mcam *mcam = &rvu->hw->mcam; 733 int blkaddr, index; 734 735 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 736 if (blkaddr < 0) 737 return; 738 739 /* Get 'pcifunc' of PF device */ 740 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 741 742 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 743 nixlf, NIXLF_PROMISC_ENTRY); 744 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 745 } 746 747 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 748 int nixlf, u64 chan) 749 { 750 struct rvu_pfvf *pfvf; 751 struct npc_install_flow_req req = { 0 }; 752 struct npc_install_flow_rsp rsp = { 0 }; 753 struct npc_mcam *mcam = &rvu->hw->mcam; 754 struct rvu_hwinfo *hw = rvu->hw; 755 int blkaddr, index; 756 757 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 758 if (blkaddr < 0) 759 return; 760 761 /* Skip LBK VFs */ 762 if (is_afvf(pcifunc)) 763 return; 764 765 /* If pkt replication is not supported, 766 * then only PF is allowed to add a bcast match entry. 767 */ 768 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 769 return; 770 771 /* Get 'pcifunc' of PF device */ 772 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 773 pfvf = rvu_get_pfvf(rvu, pcifunc); 774 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 775 nixlf, NIXLF_BCAST_ENTRY); 776 777 if (!hw->cap.nix_rx_multicast) { 778 /* Early silicon doesn't support pkt replication, 779 * so install entry with UCAST action, so that PF 780 * receives all broadcast packets. 781 */ 782 req.op = NIX_RX_ACTIONOP_UCAST; 783 } else { 784 req.op = NIX_RX_ACTIONOP_MCAST; 785 req.index = pfvf->bcast_mce_idx; 786 } 787 788 eth_broadcast_addr((u8 *)&req.packet.dmac); 789 eth_broadcast_addr((u8 *)&req.mask.dmac); 790 req.features = BIT_ULL(NPC_DMAC); 791 req.channel = chan; 792 req.chan_mask = 0xFFFU; 793 req.intf = pfvf->nix_rx_intf; 794 req.entry = index; 795 req.hdr.pcifunc = 0; /* AF is requester */ 796 req.vf = pcifunc; 797 798 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 799 } 800 801 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 802 bool enable) 803 { 804 struct npc_mcam *mcam = &rvu->hw->mcam; 805 int blkaddr, index; 806 807 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 808 if (blkaddr < 0) 809 return; 810 811 /* Get 'pcifunc' of PF device */ 812 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 813 814 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 815 NIXLF_BCAST_ENTRY); 816 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 817 } 818 819 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 820 u64 chan) 821 { 822 struct npc_install_flow_req req = { 0 }; 823 struct npc_install_flow_rsp rsp = { 0 }; 824 struct npc_mcam *mcam = &rvu->hw->mcam; 825 struct rvu_hwinfo *hw = rvu->hw; 826 int blkaddr, ucast_idx, index; 827 u8 mac_addr[ETH_ALEN] = { 0 }; 828 struct nix_rx_action action; 829 struct rvu_pfvf *pfvf; 830 u16 vf_func; 831 832 /* Only CGX PF/VF can add allmulticast entry */ 833 if (is_afvf(pcifunc) && is_sdp_vf(pcifunc)) 834 return; 835 836 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 837 if (blkaddr < 0) 838 return; 839 840 /* Get 'pcifunc' of PF device */ 841 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 842 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 843 pfvf = rvu_get_pfvf(rvu, pcifunc); 844 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 845 nixlf, NIXLF_ALLMULTI_ENTRY); 846 847 /* If the corresponding PF's ucast action is RSS, 848 * use the same action for multicast entry also 849 */ 850 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 851 nixlf, NIXLF_UCAST_ENTRY); 852 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 853 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 854 blkaddr, ucast_idx); 855 856 if (action.op != NIX_RX_ACTIONOP_RSS) { 857 *(u64 *)&action = 0x00; 858 action.op = NIX_RX_ACTIONOP_UCAST; 859 action.pf_func = pcifunc; 860 } 861 862 /* RX_ACTION set to MCAST for CGX PF's */ 863 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 864 *(u64 *)&action = 0x00; 865 action.op = NIX_RX_ACTIONOP_MCAST; 866 action.index = pfvf->mcast_mce_idx; 867 } 868 869 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 870 ether_addr_copy(req.packet.dmac, mac_addr); 871 ether_addr_copy(req.mask.dmac, mac_addr); 872 req.features = BIT_ULL(NPC_DMAC); 873 874 /* For cn10k the upper two bits of the channel number are 875 * cpt channel number. with masking out these bits in the 876 * mcam entry, same entry used for NIX will allow packets 877 * received from cpt for parsing. 878 */ 879 if (!is_rvu_otx2(rvu)) 880 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 881 else 882 req.chan_mask = 0xFFFU; 883 884 req.channel = chan; 885 req.intf = pfvf->nix_rx_intf; 886 req.entry = index; 887 req.op = action.op; 888 req.hdr.pcifunc = 0; /* AF is requester */ 889 req.vf = pcifunc | vf_func; 890 req.index = action.index; 891 req.match_id = action.match_id; 892 req.flow_key_alg = action.flow_key_alg; 893 894 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 895 } 896 897 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 898 bool enable) 899 { 900 struct npc_mcam *mcam = &rvu->hw->mcam; 901 int blkaddr, index; 902 903 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 904 if (blkaddr < 0) 905 return; 906 907 /* Get 'pcifunc' of PF device */ 908 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 909 910 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 911 NIXLF_ALLMULTI_ENTRY); 912 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 913 } 914 915 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 916 int blkaddr, u16 pcifunc, u64 rx_action) 917 { 918 int actindex, index, bank, entry; 919 bool enable; 920 921 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 922 return; 923 924 mutex_lock(&mcam->lock); 925 for (index = 0; index < mcam->bmap_entries; index++) { 926 if (mcam->entry2target_pffunc[index] == pcifunc) { 927 bank = npc_get_bank(mcam, index); 928 actindex = index; 929 entry = index & (mcam->banksize - 1); 930 931 /* read vf flow entry enable status */ 932 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 933 actindex); 934 /* disable before mcam entry update */ 935 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 936 false); 937 /* update 'action' */ 938 rvu_write64(rvu, blkaddr, 939 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 940 rx_action); 941 if (enable) 942 npc_enable_mcam_entry(rvu, mcam, blkaddr, 943 actindex, true); 944 } 945 } 946 mutex_unlock(&mcam->lock); 947 } 948 949 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 950 int group, int alg_idx, int mcam_index) 951 { 952 struct npc_mcam *mcam = &rvu->hw->mcam; 953 struct rvu_hwinfo *hw = rvu->hw; 954 struct nix_rx_action action; 955 int blkaddr, index, bank; 956 struct rvu_pfvf *pfvf; 957 958 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 959 if (blkaddr < 0) 960 return; 961 962 /* Check if this is for reserved default entry */ 963 if (mcam_index < 0) { 964 if (group != DEFAULT_RSS_CONTEXT_GROUP) 965 return; 966 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 967 nixlf, NIXLF_UCAST_ENTRY); 968 } else { 969 /* TODO: validate this mcam index */ 970 index = mcam_index; 971 } 972 973 if (index >= mcam->total_entries) 974 return; 975 976 bank = npc_get_bank(mcam, index); 977 index &= (mcam->banksize - 1); 978 979 *(u64 *)&action = rvu_read64(rvu, blkaddr, 980 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 981 /* Ignore if no action was set earlier */ 982 if (!*(u64 *)&action) 983 return; 984 985 action.op = NIX_RX_ACTIONOP_RSS; 986 action.pf_func = pcifunc; 987 action.index = group; 988 action.flow_key_alg = alg_idx; 989 990 rvu_write64(rvu, blkaddr, 991 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 992 993 /* update the VF flow rule action with the VF default entry action */ 994 if (mcam_index < 0) 995 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 996 *(u64 *)&action); 997 998 /* update the action change in default rule */ 999 pfvf = rvu_get_pfvf(rvu, pcifunc); 1000 if (pfvf->def_ucast_rule) 1001 pfvf->def_ucast_rule->rx_action = action; 1002 1003 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1004 nixlf, NIXLF_PROMISC_ENTRY); 1005 1006 /* If PF's promiscuous entry is enabled, 1007 * Set RSS action for that entry as well 1008 */ 1009 if ((!hw->cap.nix_rx_multicast || !pfvf->use_mce_list) && 1010 is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 1011 bank = npc_get_bank(mcam, index); 1012 index &= (mcam->banksize - 1); 1013 1014 rvu_write64(rvu, blkaddr, 1015 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 1016 *(u64 *)&action); 1017 } 1018 } 1019 1020 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1021 int nixlf, int type, bool enable) 1022 { 1023 struct npc_mcam *mcam = &rvu->hw->mcam; 1024 struct rvu_hwinfo *hw = rvu->hw; 1025 struct nix_mce_list *mce_list; 1026 int index, blkaddr, mce_idx; 1027 struct rvu_pfvf *pfvf; 1028 1029 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1030 if (blkaddr < 0) 1031 return; 1032 1033 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1034 nixlf, type); 1035 1036 /* disable MCAM entry when packet replication is not supported by hw */ 1037 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1038 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1039 return; 1040 } 1041 1042 /* return incase mce list is not enabled */ 1043 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1044 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1045 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1046 return; 1047 1048 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1049 1050 nix_update_mce_list(rvu, pcifunc, mce_list, 1051 mce_idx, index, enable); 1052 if (enable) 1053 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1054 } 1055 1056 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1057 int nixlf, bool enable) 1058 { 1059 struct npc_mcam *mcam = &rvu->hw->mcam; 1060 int index, blkaddr; 1061 1062 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1063 if (blkaddr < 0) 1064 return; 1065 1066 /* Ucast MCAM match entry of this PF/VF */ 1067 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1068 nixlf, NIXLF_UCAST_ENTRY); 1069 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1070 1071 /* Nothing to do for VFs, on platforms where pkt replication 1072 * is not supported 1073 */ 1074 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1075 return; 1076 1077 /* add/delete pf_func to broadcast MCE list */ 1078 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1079 NIXLF_BCAST_ENTRY, enable); 1080 } 1081 1082 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1083 { 1084 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1085 1086 /* Delete multicast and promisc MCAM entries */ 1087 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1088 NIXLF_ALLMULTI_ENTRY, false); 1089 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1090 NIXLF_PROMISC_ENTRY, false); 1091 } 1092 1093 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1094 { 1095 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1096 * in set_rx_mode mbox handler. 1097 */ 1098 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1099 } 1100 1101 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1102 { 1103 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1104 struct npc_mcam *mcam = &rvu->hw->mcam; 1105 struct rvu_npc_mcam_rule *rule, *tmp; 1106 int blkaddr; 1107 1108 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1109 if (blkaddr < 0) 1110 return; 1111 1112 mutex_lock(&mcam->lock); 1113 1114 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1115 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1116 if (is_npc_intf_rx(rule->intf) && 1117 rule->rx_action.pf_func == pcifunc && 1118 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1119 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1120 rule->entry, false); 1121 rule->enable = false; 1122 /* Indicate that default rule is disabled */ 1123 if (rule->default_rule) { 1124 pfvf->def_ucast_rule = NULL; 1125 list_del(&rule->list); 1126 kfree(rule); 1127 } 1128 } 1129 } 1130 1131 mutex_unlock(&mcam->lock); 1132 1133 npc_mcam_disable_flows(rvu, pcifunc); 1134 1135 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1136 } 1137 1138 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1139 { 1140 struct npc_mcam *mcam = &rvu->hw->mcam; 1141 struct rvu_npc_mcam_rule *rule, *tmp; 1142 int blkaddr; 1143 1144 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1145 if (blkaddr < 0) 1146 return; 1147 1148 mutex_lock(&mcam->lock); 1149 1150 /* Free all MCAM entries owned by this 'pcifunc' */ 1151 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1152 1153 /* Free all MCAM counters owned by this 'pcifunc' */ 1154 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1155 1156 /* Delete MCAM entries owned by this 'pcifunc' */ 1157 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1158 if (rule->owner == pcifunc && !rule->default_rule) { 1159 list_del(&rule->list); 1160 kfree(rule); 1161 } 1162 } 1163 1164 mutex_unlock(&mcam->lock); 1165 1166 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1167 } 1168 1169 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1170 rvu_write64(rvu, blkaddr, \ 1171 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1172 1173 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1174 rvu_write64(rvu, blkaddr, \ 1175 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1176 1177 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1178 struct npc_mcam_kex *mkex, u8 intf) 1179 { 1180 int lid, lt, ld, fl; 1181 1182 if (is_npc_intf_tx(intf)) 1183 return; 1184 1185 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1186 mkex->keyx_cfg[NIX_INTF_RX]); 1187 1188 /* Program LDATA */ 1189 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1190 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1191 for (ld = 0; ld < NPC_MAX_LD; ld++) 1192 SET_KEX_LD(intf, lid, lt, ld, 1193 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1194 [lid][lt][ld]); 1195 } 1196 } 1197 /* Program LFLAGS */ 1198 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1199 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1200 SET_KEX_LDFLAGS(intf, ld, fl, 1201 mkex->intf_ld_flags[NIX_INTF_RX] 1202 [ld][fl]); 1203 } 1204 } 1205 1206 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1207 struct npc_mcam_kex *mkex, u8 intf) 1208 { 1209 int lid, lt, ld, fl; 1210 1211 if (is_npc_intf_rx(intf)) 1212 return; 1213 1214 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1215 mkex->keyx_cfg[NIX_INTF_TX]); 1216 1217 /* Program LDATA */ 1218 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1219 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1220 for (ld = 0; ld < NPC_MAX_LD; ld++) 1221 SET_KEX_LD(intf, lid, lt, ld, 1222 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1223 [lid][lt][ld]); 1224 } 1225 } 1226 /* Program LFLAGS */ 1227 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1228 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1229 SET_KEX_LDFLAGS(intf, ld, fl, 1230 mkex->intf_ld_flags[NIX_INTF_TX] 1231 [ld][fl]); 1232 } 1233 } 1234 1235 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1236 struct npc_mcam_kex *mkex) 1237 { 1238 struct rvu_hwinfo *hw = rvu->hw; 1239 u8 intf; 1240 int ld; 1241 1242 for (ld = 0; ld < NPC_MAX_LD; ld++) 1243 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1244 mkex->kex_ld_flags[ld]); 1245 1246 for (intf = 0; intf < hw->npc_intfs; intf++) { 1247 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1248 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1249 } 1250 } 1251 1252 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1253 u64 *size) 1254 { 1255 u64 prfl_addr, prfl_sz; 1256 1257 if (!rvu->fwdata) 1258 return -EINVAL; 1259 1260 prfl_addr = rvu->fwdata->mcam_addr; 1261 prfl_sz = rvu->fwdata->mcam_sz; 1262 1263 if (!prfl_addr || !prfl_sz) 1264 return -EINVAL; 1265 1266 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1267 if (!(*prfl_img_addr)) 1268 return -ENOMEM; 1269 1270 *size = prfl_sz; 1271 1272 return 0; 1273 } 1274 1275 /* strtoull of "mkexprof" with base:36 */ 1276 #define MKEX_END_SIGN 0xdeadbeef 1277 1278 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1279 const char *mkex_profile) 1280 { 1281 struct device *dev = &rvu->pdev->dev; 1282 struct npc_mcam_kex *mcam_kex; 1283 void __iomem *mkex_prfl_addr = NULL; 1284 u64 prfl_sz; 1285 int ret; 1286 1287 /* If user not selected mkex profile */ 1288 if (rvu->kpu_fwdata_sz || 1289 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1290 goto program_mkex; 1291 1292 /* Setting up the mapping for mkex profile image */ 1293 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1294 if (ret < 0) 1295 goto program_mkex; 1296 1297 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1298 1299 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1300 /* Compare with mkex mod_param name string */ 1301 if (mcam_kex->mkex_sign == MKEX_SIGN && 1302 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1303 /* Due to an errata (35786) in A0/B0 pass silicon, 1304 * parse nibble enable configuration has to be 1305 * identical for both Rx and Tx interfaces. 1306 */ 1307 if (!is_rvu_96xx_B0(rvu) || 1308 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1309 rvu->kpu.mkex = mcam_kex; 1310 goto program_mkex; 1311 } 1312 1313 mcam_kex++; 1314 prfl_sz -= sizeof(struct npc_mcam_kex); 1315 } 1316 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1317 1318 program_mkex: 1319 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1320 /* Program selected mkex profile */ 1321 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1322 if (mkex_prfl_addr) 1323 iounmap(mkex_prfl_addr); 1324 } 1325 1326 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1327 const struct npc_kpu_profile_action *kpuaction, 1328 int kpu, int entry, bool pkind) 1329 { 1330 struct npc_kpu_action0 action0 = {0}; 1331 struct npc_kpu_action1 action1 = {0}; 1332 u64 reg; 1333 1334 action1.errlev = kpuaction->errlev; 1335 action1.errcode = kpuaction->errcode; 1336 action1.dp0_offset = kpuaction->dp0_offset; 1337 action1.dp1_offset = kpuaction->dp1_offset; 1338 action1.dp2_offset = kpuaction->dp2_offset; 1339 1340 if (pkind) 1341 reg = NPC_AF_PKINDX_ACTION1(entry); 1342 else 1343 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1344 1345 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1346 1347 action0.byp_count = kpuaction->bypass_count; 1348 action0.capture_ena = kpuaction->cap_ena; 1349 action0.parse_done = kpuaction->parse_done; 1350 action0.next_state = kpuaction->next_state; 1351 action0.capture_lid = kpuaction->lid; 1352 action0.capture_ltype = kpuaction->ltype; 1353 action0.capture_flags = kpuaction->flags; 1354 action0.ptr_advance = kpuaction->ptr_advance; 1355 action0.var_len_offset = kpuaction->offset; 1356 action0.var_len_mask = kpuaction->mask; 1357 action0.var_len_right = kpuaction->right; 1358 action0.var_len_shift = kpuaction->shift; 1359 1360 if (pkind) 1361 reg = NPC_AF_PKINDX_ACTION0(entry); 1362 else 1363 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1364 1365 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1366 } 1367 1368 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1369 const struct npc_kpu_profile_cam *kpucam, 1370 int kpu, int entry) 1371 { 1372 struct npc_kpu_cam cam0 = {0}; 1373 struct npc_kpu_cam cam1 = {0}; 1374 1375 cam1.state = kpucam->state & kpucam->state_mask; 1376 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1377 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1378 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1379 1380 cam0.state = ~kpucam->state & kpucam->state_mask; 1381 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1382 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1383 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1384 1385 rvu_write64(rvu, blkaddr, 1386 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1387 rvu_write64(rvu, blkaddr, 1388 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1389 } 1390 1391 static inline u64 enable_mask(int count) 1392 { 1393 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1394 } 1395 1396 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1397 const struct npc_kpu_profile *profile) 1398 { 1399 int entry, num_entries, max_entries; 1400 u64 entry_mask; 1401 1402 if (profile->cam_entries != profile->action_entries) { 1403 dev_err(rvu->dev, 1404 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1405 kpu, profile->cam_entries, profile->action_entries); 1406 } 1407 1408 max_entries = rvu->hw->npc_kpu_entries; 1409 1410 /* Program CAM match entries for previous KPU extracted data */ 1411 num_entries = min_t(int, profile->cam_entries, max_entries); 1412 for (entry = 0; entry < num_entries; entry++) 1413 npc_config_kpucam(rvu, blkaddr, 1414 &profile->cam[entry], kpu, entry); 1415 1416 /* Program this KPU's actions */ 1417 num_entries = min_t(int, profile->action_entries, max_entries); 1418 for (entry = 0; entry < num_entries; entry++) 1419 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1420 kpu, entry, false); 1421 1422 /* Enable all programmed entries */ 1423 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1424 entry_mask = enable_mask(num_entries); 1425 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1426 if (!rvu->kpu.custom) 1427 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1428 rvu_write64(rvu, blkaddr, 1429 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1430 if (num_entries > 64) { 1431 rvu_write64(rvu, blkaddr, 1432 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1433 enable_mask(num_entries - 64)); 1434 } 1435 1436 /* Enable this KPU */ 1437 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1438 } 1439 1440 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1441 { 1442 profile->custom = 0; 1443 profile->name = def_pfl_name; 1444 profile->version = NPC_KPU_PROFILE_VER; 1445 profile->ikpu = ikpu_action_entries; 1446 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1447 profile->kpu = npc_kpu_profiles; 1448 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1449 profile->lt_def = &npc_lt_defaults; 1450 profile->mkex = &npc_mkex_default; 1451 1452 return 0; 1453 } 1454 1455 static int npc_apply_custom_kpu(struct rvu *rvu, 1456 struct npc_kpu_profile_adapter *profile) 1457 { 1458 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1459 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1460 struct npc_kpu_profile_action *action; 1461 struct npc_kpu_profile_cam *cam; 1462 struct npc_kpu_fwdata *fw_kpu; 1463 int entries; 1464 u16 kpu, entry; 1465 1466 if (rvu->kpu_fwdata_sz < hdr_sz) { 1467 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1468 return -EINVAL; 1469 } 1470 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1471 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1472 fw->signature); 1473 return -EINVAL; 1474 } 1475 /* Verify if the using known profile structure */ 1476 if (NPC_KPU_VER_MAJ(profile->version) > 1477 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1478 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1479 NPC_KPU_VER_MAJ(profile->version), 1480 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1481 return -EINVAL; 1482 } 1483 /* Verify if profile is aligned with the required kernel changes */ 1484 if (NPC_KPU_VER_MIN(profile->version) < 1485 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1486 dev_warn(rvu->dev, 1487 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1488 NPC_KPU_VER_MAJ(profile->version), 1489 NPC_KPU_VER_MIN(profile->version), 1490 NPC_KPU_VER_PATCH(profile->version), 1491 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1492 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1493 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1494 return -EINVAL; 1495 } 1496 /* Verify if profile fits the HW */ 1497 if (fw->kpus > profile->kpus) { 1498 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1499 profile->kpus); 1500 return -EINVAL; 1501 } 1502 1503 profile->custom = 1; 1504 profile->name = fw->name; 1505 profile->version = le64_to_cpu(fw->version); 1506 profile->mkex = &fw->mkex; 1507 profile->lt_def = &fw->lt_def; 1508 1509 for (kpu = 0; kpu < fw->kpus; kpu++) { 1510 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1511 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1512 dev_warn(rvu->dev, 1513 "Too many custom entries on KPU%d: %d > %d\n", 1514 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1515 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1516 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1517 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1518 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1519 offset += fw_kpu->entries * sizeof(*action); 1520 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1521 dev_warn(rvu->dev, 1522 "Profile size mismatch on KPU%i parsing.\n", 1523 kpu + 1); 1524 return -EINVAL; 1525 } 1526 for (entry = 0; entry < entries; entry++) { 1527 profile->kpu[kpu].cam[entry] = cam[entry]; 1528 profile->kpu[kpu].action[entry] = action[entry]; 1529 } 1530 } 1531 1532 return 0; 1533 } 1534 1535 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1536 u64 prfl_sz, const char *kpu_profile) 1537 { 1538 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1539 int rc = -EINVAL; 1540 1541 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1542 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1543 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1544 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1545 kpu_profile); 1546 rvu->kpu_fwdata = kpu_data; 1547 rvu->kpu_fwdata_sz = prfl_sz; 1548 rvu->kpu_prfl_addr = prfl_addr; 1549 rc = 0; 1550 } 1551 1552 return rc; 1553 } 1554 1555 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1556 const char *kpu_profile) 1557 { 1558 struct npc_coalesced_kpu_prfl *img_data = NULL; 1559 int i = 0, rc = -EINVAL; 1560 void __iomem *kpu_prfl_addr; 1561 u16 offset; 1562 1563 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1564 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1565 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1566 /* Loaded profile is a single KPU profile. */ 1567 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1568 prfl_sz, kpu_profile); 1569 goto done; 1570 } 1571 1572 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1573 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1574 (img_data->num_prfl * sizeof(uint16_t)); 1575 /* Check if mapped image is coalesced image. */ 1576 while (i < img_data->num_prfl) { 1577 /* Profile image offsets are rounded up to next 8 multiple.*/ 1578 offset = ALIGN_8B_CEIL(offset); 1579 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1580 offset); 1581 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1582 img_data->prfl_sz[i], kpu_profile); 1583 if (!rc) 1584 break; 1585 /* Calculating offset of profile image based on profile size.*/ 1586 offset += img_data->prfl_sz[i]; 1587 i++; 1588 } 1589 done: 1590 return rc; 1591 } 1592 1593 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1594 { 1595 int ret = -EINVAL; 1596 u64 prfl_sz; 1597 1598 /* Setting up the mapping for NPC profile image */ 1599 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1600 if (ret < 0) 1601 goto done; 1602 1603 /* Detect if profile is coalesced or single KPU profile and load */ 1604 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1605 if (ret == 0) 1606 goto done; 1607 1608 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1609 if (rvu->kpu_prfl_addr) { 1610 iounmap(rvu->kpu_prfl_addr); 1611 rvu->kpu_prfl_addr = NULL; 1612 rvu->kpu_fwdata_sz = 0; 1613 rvu->kpu_fwdata = NULL; 1614 } 1615 1616 done: 1617 return ret; 1618 } 1619 1620 static void npc_load_kpu_profile(struct rvu *rvu) 1621 { 1622 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1623 const char *kpu_profile = rvu->kpu_pfl_name; 1624 const struct firmware *fw = NULL; 1625 bool retry_fwdb = false; 1626 1627 /* If user not specified profile customization */ 1628 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1629 goto revert_to_default; 1630 /* First prepare default KPU, then we'll customize top entries. */ 1631 npc_prepare_default_kpu(profile); 1632 1633 /* Order of preceedence for load loading NPC profile (high to low) 1634 * Firmware binary in filesystem. 1635 * Firmware database method. 1636 * Default KPU profile. 1637 */ 1638 if (!request_firmware(&fw, kpu_profile, rvu->dev)) { 1639 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1640 kpu_profile); 1641 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1642 if (rvu->kpu_fwdata) { 1643 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1644 rvu->kpu_fwdata_sz = fw->size; 1645 } 1646 release_firmware(fw); 1647 retry_fwdb = true; 1648 goto program_kpu; 1649 } 1650 1651 load_image_fwdb: 1652 /* Loading the KPU profile using firmware database */ 1653 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1654 goto revert_to_default; 1655 1656 program_kpu: 1657 /* Apply profile customization if firmware was loaded. */ 1658 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1659 /* If image from firmware filesystem fails to load or invalid 1660 * retry with firmware database method. 1661 */ 1662 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1663 /* Loading image from firmware database failed. */ 1664 if (rvu->kpu_prfl_addr) { 1665 iounmap(rvu->kpu_prfl_addr); 1666 rvu->kpu_prfl_addr = NULL; 1667 } else { 1668 kfree(rvu->kpu_fwdata); 1669 } 1670 rvu->kpu_fwdata = NULL; 1671 rvu->kpu_fwdata_sz = 0; 1672 if (retry_fwdb) { 1673 retry_fwdb = false; 1674 goto load_image_fwdb; 1675 } 1676 } 1677 1678 dev_warn(rvu->dev, 1679 "Can't load KPU profile %s. Using default.\n", 1680 kpu_profile); 1681 kfree(rvu->kpu_fwdata); 1682 rvu->kpu_fwdata = NULL; 1683 goto revert_to_default; 1684 } 1685 1686 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1687 profile->name, NPC_KPU_VER_MAJ(profile->version), 1688 NPC_KPU_VER_MIN(profile->version), 1689 NPC_KPU_VER_PATCH(profile->version)); 1690 1691 return; 1692 1693 revert_to_default: 1694 npc_prepare_default_kpu(profile); 1695 } 1696 1697 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1698 { 1699 struct rvu_hwinfo *hw = rvu->hw; 1700 int num_pkinds, num_kpus, idx; 1701 1702 /* Disable all KPUs and their entries */ 1703 for (idx = 0; idx < hw->npc_kpus; idx++) { 1704 rvu_write64(rvu, blkaddr, 1705 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1706 rvu_write64(rvu, blkaddr, 1707 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1708 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1709 } 1710 1711 /* Load and customize KPU profile. */ 1712 npc_load_kpu_profile(rvu); 1713 1714 /* First program IKPU profile i.e PKIND configs. 1715 * Check HW max count to avoid configuring junk or 1716 * writing to unsupported CSR addresses. 1717 */ 1718 num_pkinds = rvu->kpu.pkinds; 1719 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 1720 1721 for (idx = 0; idx < num_pkinds; idx++) 1722 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1723 1724 /* Program KPU CAM and Action profiles */ 1725 num_kpus = rvu->kpu.kpus; 1726 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1727 1728 for (idx = 0; idx < num_kpus; idx++) 1729 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1730 } 1731 1732 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1733 { 1734 int nixlf_count = rvu_get_nixlf_count(rvu); 1735 struct npc_mcam *mcam = &rvu->hw->mcam; 1736 int rsvd, err; 1737 u16 index; 1738 int cntr; 1739 u64 cfg; 1740 1741 /* Actual number of MCAM entries vary by entry size */ 1742 cfg = (rvu_read64(rvu, blkaddr, 1743 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1744 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1745 mcam->keysize = cfg; 1746 1747 /* Number of banks combined per MCAM entry */ 1748 if (cfg == NPC_MCAM_KEY_X4) 1749 mcam->banks_per_entry = 4; 1750 else if (cfg == NPC_MCAM_KEY_X2) 1751 mcam->banks_per_entry = 2; 1752 else 1753 mcam->banks_per_entry = 1; 1754 1755 /* Reserve one MCAM entry for each of the NIX LF to 1756 * guarantee space to install default matching DMAC rule. 1757 * Also reserve 2 MCAM entries for each PF for default 1758 * channel based matching or 'bcast & promisc' matching to 1759 * support BCAST and PROMISC modes of operation for PFs. 1760 * PF0 is excluded. 1761 */ 1762 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1763 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1764 if (mcam->total_entries <= rsvd) { 1765 dev_warn(rvu->dev, 1766 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1767 mcam->total_entries); 1768 return -ENOMEM; 1769 } 1770 1771 mcam->bmap_entries = mcam->total_entries - rsvd; 1772 mcam->nixlf_offset = mcam->bmap_entries; 1773 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1774 1775 /* Allocate bitmaps for managing MCAM entries */ 1776 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1777 sizeof(long), GFP_KERNEL); 1778 if (!mcam->bmap) 1779 return -ENOMEM; 1780 1781 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1782 BITS_TO_LONGS(mcam->bmap_entries), 1783 sizeof(long), GFP_KERNEL); 1784 if (!mcam->bmap_reverse) 1785 return -ENOMEM; 1786 1787 mcam->bmap_fcnt = mcam->bmap_entries; 1788 1789 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1790 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1791 sizeof(u16), GFP_KERNEL); 1792 if (!mcam->entry2pfvf_map) 1793 return -ENOMEM; 1794 1795 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1796 * allocations and another 1/8th at the top for high priority 1797 * allocations. 1798 */ 1799 mcam->lprio_count = mcam->bmap_entries / 8; 1800 if (mcam->lprio_count > BITS_PER_LONG) 1801 mcam->lprio_count = round_down(mcam->lprio_count, 1802 BITS_PER_LONG); 1803 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1804 mcam->hprio_count = mcam->lprio_count; 1805 mcam->hprio_end = mcam->hprio_count; 1806 1807 1808 /* Allocate bitmap for managing MCAM counters and memory 1809 * for saving counter to RVU PFFUNC allocation mapping. 1810 */ 1811 err = rvu_alloc_bitmap(&mcam->counters); 1812 if (err) 1813 return err; 1814 1815 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1816 sizeof(u16), GFP_KERNEL); 1817 if (!mcam->cntr2pfvf_map) 1818 goto free_mem; 1819 1820 /* Alloc memory for MCAM entry to counter mapping and for tracking 1821 * counter's reference count. 1822 */ 1823 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1824 sizeof(u16), GFP_KERNEL); 1825 if (!mcam->entry2cntr_map) 1826 goto free_mem; 1827 1828 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1829 sizeof(u16), GFP_KERNEL); 1830 if (!mcam->cntr_refcnt) 1831 goto free_mem; 1832 1833 /* Alloc memory for saving target device of mcam rule */ 1834 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1835 sizeof(u16), GFP_KERNEL); 1836 if (!mcam->entry2target_pffunc) 1837 goto free_mem; 1838 1839 for (index = 0; index < mcam->bmap_entries; index++) { 1840 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1841 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1842 } 1843 1844 for (cntr = 0; cntr < mcam->counters.max; cntr++) 1845 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1846 1847 mutex_init(&mcam->lock); 1848 1849 return 0; 1850 1851 free_mem: 1852 kfree(mcam->counters.bmap); 1853 return -ENOMEM; 1854 } 1855 1856 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1857 { 1858 struct npc_pkind *pkind = &rvu->hw->pkind; 1859 struct npc_mcam *mcam = &rvu->hw->mcam; 1860 struct rvu_hwinfo *hw = rvu->hw; 1861 u64 npc_const, npc_const1; 1862 u64 npc_const2 = 0; 1863 1864 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1865 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1866 if (npc_const1 & BIT_ULL(63)) 1867 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1868 1869 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 1870 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 1871 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1872 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1873 hw->npc_intfs = npc_const & 0xFULL; 1874 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1875 1876 mcam->banks = (npc_const >> 44) & 0xFULL; 1877 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1878 hw->npc_stat_ena = BIT_ULL(9); 1879 /* Extended set */ 1880 if (npc_const2) { 1881 hw->npc_ext_set = true; 1882 /* 96xx supports only match_stats and npc_counters 1883 * reflected in NPC_AF_CONST reg. 1884 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 1885 * 98xx has both match_stat and ext and npc_counter 1886 * reflected in NPC_AF_CONST2 1887 * STAT_SEL_EXT added at [12:14] bit position. 1888 * cn10k supports only ext and hence npc_counters in 1889 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 1890 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 1891 */ 1892 if (!hw->npc_counters) 1893 hw->npc_stat_ena = BIT_ULL(63); 1894 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1895 mcam->banksize = npc_const2 & 0xFFFFULL; 1896 } 1897 1898 mcam->counters.max = hw->npc_counters; 1899 } 1900 1901 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1902 { 1903 struct npc_mcam *mcam = &rvu->hw->mcam; 1904 struct rvu_hwinfo *hw = rvu->hw; 1905 u64 nibble_ena, rx_kex, tx_kex; 1906 u8 intf; 1907 1908 /* Reserve last counter for MCAM RX miss action which is set to 1909 * drop packet. This way we will know how many pkts didn't match 1910 * any MCAM entry. 1911 */ 1912 mcam->counters.max--; 1913 mcam->rx_miss_act_cntr = mcam->counters.max; 1914 1915 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1916 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1917 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1918 1919 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1920 if (nibble_ena) { 1921 tx_kex &= ~NPC_PARSE_NIBBLE; 1922 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1923 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1924 } 1925 1926 /* Configure RX interfaces */ 1927 for (intf = 0; intf < hw->npc_intfs; intf++) { 1928 if (is_npc_intf_tx(intf)) 1929 continue; 1930 1931 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1932 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1933 rx_kex); 1934 1935 /* If MCAM lookup doesn't result in a match, drop the received 1936 * packet. And map this action to a counter to count dropped 1937 * packets. 1938 */ 1939 rvu_write64(rvu, blkaddr, 1940 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1941 1942 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1943 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1944 */ 1945 rvu_write64(rvu, blkaddr, 1946 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1947 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1948 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 1949 } 1950 1951 /* Configure TX interfaces */ 1952 for (intf = 0; intf < hw->npc_intfs; intf++) { 1953 if (is_npc_intf_rx(intf)) 1954 continue; 1955 1956 /* Extract Ltypes LID_LA to LID_LE */ 1957 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1958 tx_kex); 1959 1960 /* Set TX miss action to UCAST_DEFAULT i.e 1961 * transmit the packet on NIX LF SQ's default channel. 1962 */ 1963 rvu_write64(rvu, blkaddr, 1964 NPC_AF_INTFX_MISS_ACT(intf), 1965 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1966 } 1967 } 1968 1969 int rvu_npc_init(struct rvu *rvu) 1970 { 1971 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1972 struct npc_pkind *pkind = &rvu->hw->pkind; 1973 struct npc_mcam *mcam = &rvu->hw->mcam; 1974 int blkaddr, entry, bank, err; 1975 1976 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1977 if (blkaddr < 0) { 1978 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1979 return -ENODEV; 1980 } 1981 1982 rvu_npc_hw_init(rvu, blkaddr); 1983 1984 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1985 for (bank = 0; bank < mcam->banks; bank++) { 1986 for (entry = 0; entry < mcam->banksize; entry++) 1987 rvu_write64(rvu, blkaddr, 1988 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1989 } 1990 1991 err = rvu_alloc_bitmap(&pkind->rsrc); 1992 if (err) 1993 return err; 1994 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 1995 * no need to configure PKIND for all LBKs separately. 1996 */ 1997 rvu_alloc_rsrc(&pkind->rsrc); 1998 1999 /* Allocate mem for pkind to PF and channel mapping info */ 2000 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2001 sizeof(u32), GFP_KERNEL); 2002 if (!pkind->pfchan_map) 2003 return -ENOMEM; 2004 2005 /* Configure KPU profile */ 2006 npc_parser_profile_init(rvu, blkaddr); 2007 2008 /* Config Outer L2, IPv4's NPC layer info */ 2009 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2010 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2011 kpu->lt_def->pck_ol2.ltype_mask); 2012 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2013 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2014 kpu->lt_def->pck_oip4.ltype_mask); 2015 2016 /* Config Inner IPV4 NPC layer info */ 2017 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2018 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2019 kpu->lt_def->pck_iip4.ltype_mask); 2020 2021 /* Enable below for Rx pkts. 2022 * - Outer IPv4 header checksum validation. 2023 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2024 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2025 * - Inner IPv4 header checksum validation. 2026 * - Set non zero checksum error code value 2027 */ 2028 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2029 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2030 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2031 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2032 2033 rvu_npc_setup_interfaces(rvu, blkaddr); 2034 2035 /* Configure MKEX profile */ 2036 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2037 2038 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2039 if (err) 2040 return err; 2041 2042 err = npc_flow_steering_init(rvu, blkaddr); 2043 if (err) { 2044 dev_err(rvu->dev, 2045 "Incorrect mkex profile loaded using default mkex\n"); 2046 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2047 } 2048 2049 return 0; 2050 } 2051 2052 void rvu_npc_freemem(struct rvu *rvu) 2053 { 2054 struct npc_pkind *pkind = &rvu->hw->pkind; 2055 struct npc_mcam *mcam = &rvu->hw->mcam; 2056 2057 kfree(pkind->rsrc.bmap); 2058 kfree(mcam->counters.bmap); 2059 if (rvu->kpu_prfl_addr) 2060 iounmap(rvu->kpu_prfl_addr); 2061 else 2062 kfree(rvu->kpu_fwdata); 2063 mutex_destroy(&mcam->lock); 2064 } 2065 2066 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2067 int blkaddr, int *alloc_cnt, 2068 int *enable_cnt) 2069 { 2070 struct npc_mcam *mcam = &rvu->hw->mcam; 2071 int entry; 2072 2073 *alloc_cnt = 0; 2074 *enable_cnt = 0; 2075 2076 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2077 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2078 (*alloc_cnt)++; 2079 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2080 (*enable_cnt)++; 2081 } 2082 } 2083 } 2084 2085 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2086 int blkaddr, int *alloc_cnt, 2087 int *enable_cnt) 2088 { 2089 struct npc_mcam *mcam = &rvu->hw->mcam; 2090 int cntr; 2091 2092 *alloc_cnt = 0; 2093 *enable_cnt = 0; 2094 2095 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2096 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2097 (*alloc_cnt)++; 2098 if (mcam->cntr_refcnt[cntr]) 2099 (*enable_cnt)++; 2100 } 2101 } 2102 } 2103 2104 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 2105 u16 pcifunc, int entry) 2106 { 2107 /* verify AF installed entries */ 2108 if (is_pffunc_af(pcifunc)) 2109 return 0; 2110 /* Verify if entry is valid and if it is indeed 2111 * allocated to the requesting PFFUNC. 2112 */ 2113 if (entry >= mcam->bmap_entries) 2114 return NPC_MCAM_INVALID_REQ; 2115 2116 if (pcifunc != mcam->entry2pfvf_map[entry]) 2117 return NPC_MCAM_PERM_DENIED; 2118 2119 return 0; 2120 } 2121 2122 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2123 u16 pcifunc, int cntr) 2124 { 2125 /* Verify if counter is valid and if it is indeed 2126 * allocated to the requesting PFFUNC. 2127 */ 2128 if (cntr >= mcam->counters.max) 2129 return NPC_MCAM_INVALID_REQ; 2130 2131 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2132 return NPC_MCAM_PERM_DENIED; 2133 2134 return 0; 2135 } 2136 2137 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2138 int blkaddr, u16 entry, u16 cntr) 2139 { 2140 u16 index = entry & (mcam->banksize - 1); 2141 u32 bank = npc_get_bank(mcam, entry); 2142 struct rvu_hwinfo *hw = rvu->hw; 2143 2144 /* Set mapping and increment counter's refcnt */ 2145 mcam->entry2cntr_map[entry] = cntr; 2146 mcam->cntr_refcnt[cntr]++; 2147 /* Enable stats */ 2148 rvu_write64(rvu, blkaddr, 2149 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2150 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2151 } 2152 2153 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2154 struct npc_mcam *mcam, 2155 int blkaddr, u16 entry, u16 cntr) 2156 { 2157 u16 index = entry & (mcam->banksize - 1); 2158 u32 bank = npc_get_bank(mcam, entry); 2159 2160 /* Remove mapping and reduce counter's refcnt */ 2161 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2162 mcam->cntr_refcnt[cntr]--; 2163 /* Disable stats */ 2164 rvu_write64(rvu, blkaddr, 2165 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2166 } 2167 2168 /* Sets MCAM entry in bitmap as used. Update 2169 * reverse bitmap too. Should be called with 2170 * 'mcam->lock' held. 2171 */ 2172 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2173 { 2174 u16 entry, rentry; 2175 2176 entry = index; 2177 rentry = mcam->bmap_entries - index - 1; 2178 2179 __set_bit(entry, mcam->bmap); 2180 __set_bit(rentry, mcam->bmap_reverse); 2181 mcam->bmap_fcnt--; 2182 } 2183 2184 /* Sets MCAM entry in bitmap as free. Update 2185 * reverse bitmap too. Should be called with 2186 * 'mcam->lock' held. 2187 */ 2188 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2189 { 2190 u16 entry, rentry; 2191 2192 entry = index; 2193 rentry = mcam->bmap_entries - index - 1; 2194 2195 __clear_bit(entry, mcam->bmap); 2196 __clear_bit(rentry, mcam->bmap_reverse); 2197 mcam->bmap_fcnt++; 2198 } 2199 2200 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2201 int blkaddr, u16 pcifunc) 2202 { 2203 u16 index, cntr; 2204 2205 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2206 for (index = 0; index < mcam->bmap_entries; index++) { 2207 if (mcam->entry2pfvf_map[index] == pcifunc) { 2208 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2209 /* Free the entry in bitmap */ 2210 npc_mcam_clear_bit(mcam, index); 2211 /* Disable the entry */ 2212 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2213 2214 /* Update entry2counter mapping */ 2215 cntr = mcam->entry2cntr_map[index]; 2216 if (cntr != NPC_MCAM_INVALID_MAP) 2217 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2218 blkaddr, index, 2219 cntr); 2220 mcam->entry2target_pffunc[index] = 0x0; 2221 } 2222 } 2223 } 2224 2225 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2226 u16 pcifunc) 2227 { 2228 u16 cntr; 2229 2230 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2231 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2232 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2233 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2234 mcam->cntr_refcnt[cntr] = 0; 2235 rvu_free_rsrc(&mcam->counters, cntr); 2236 /* This API is expected to be called after freeing 2237 * MCAM entries, which inturn will remove 2238 * 'entry to counter' mapping. 2239 * No need to do it again. 2240 */ 2241 } 2242 } 2243 } 2244 2245 /* Find area of contiguous free entries of size 'nr'. 2246 * If not found return max contiguous free entries available. 2247 */ 2248 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2249 u16 nr, u16 *max_area) 2250 { 2251 u16 max_area_start = 0; 2252 u16 index, next, end; 2253 2254 *max_area = 0; 2255 2256 again: 2257 index = find_next_zero_bit(map, size, start); 2258 if (index >= size) 2259 return max_area_start; 2260 2261 end = ((index + nr) >= size) ? size : index + nr; 2262 next = find_next_bit(map, end, index); 2263 if (*max_area < (next - index)) { 2264 *max_area = next - index; 2265 max_area_start = index; 2266 } 2267 2268 if (next < end) { 2269 start = next + 1; 2270 goto again; 2271 } 2272 2273 return max_area_start; 2274 } 2275 2276 /* Find number of free MCAM entries available 2277 * within range i.e in between 'start' and 'end'. 2278 */ 2279 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2280 { 2281 u16 index, next; 2282 u16 fcnt = 0; 2283 2284 again: 2285 if (start >= end) 2286 return fcnt; 2287 2288 index = find_next_zero_bit(map, end, start); 2289 if (index >= end) 2290 return fcnt; 2291 2292 next = find_next_bit(map, end, index); 2293 if (next <= end) { 2294 fcnt += next - index; 2295 start = next + 1; 2296 goto again; 2297 } 2298 2299 fcnt += end - index; 2300 return fcnt; 2301 } 2302 2303 static void 2304 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2305 struct npc_mcam_alloc_entry_req *req, 2306 u16 *start, u16 *end, bool *reverse) 2307 { 2308 u16 fcnt; 2309 2310 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2311 goto hprio; 2312 2313 /* For a low priority entry allocation 2314 * - If reference entry is not in hprio zone then 2315 * search range: ref_entry to end. 2316 * - If reference entry is in hprio zone and if 2317 * request can be accomodated in non-hprio zone then 2318 * search range: 'start of middle zone' to 'end' 2319 * - else search in reverse, so that less number of hprio 2320 * zone entries are allocated. 2321 */ 2322 2323 *reverse = false; 2324 *start = req->ref_entry + 1; 2325 *end = mcam->bmap_entries; 2326 2327 if (req->ref_entry >= mcam->hprio_end) 2328 return; 2329 2330 fcnt = npc_mcam_get_free_count(mcam->bmap, 2331 mcam->hprio_end, mcam->bmap_entries); 2332 if (fcnt > req->count) 2333 *start = mcam->hprio_end; 2334 else 2335 *reverse = true; 2336 return; 2337 2338 hprio: 2339 /* For a high priority entry allocation, search is always 2340 * in reverse to preserve hprio zone entries. 2341 * - If reference entry is not in lprio zone then 2342 * search range: 0 to ref_entry. 2343 * - If reference entry is in lprio zone and if 2344 * request can be accomodated in middle zone then 2345 * search range: 'hprio_end' to 'lprio_start' 2346 */ 2347 2348 *reverse = true; 2349 *start = 0; 2350 *end = req->ref_entry; 2351 2352 if (req->ref_entry <= mcam->lprio_start) 2353 return; 2354 2355 fcnt = npc_mcam_get_free_count(mcam->bmap, 2356 mcam->hprio_end, mcam->lprio_start); 2357 if (fcnt < req->count) 2358 return; 2359 *start = mcam->hprio_end; 2360 *end = mcam->lprio_start; 2361 } 2362 2363 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2364 struct npc_mcam_alloc_entry_req *req, 2365 struct npc_mcam_alloc_entry_rsp *rsp) 2366 { 2367 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2368 u16 fcnt, hp_fcnt, lp_fcnt; 2369 u16 start, end, index; 2370 int entry, next_start; 2371 bool reverse = false; 2372 unsigned long *bmap; 2373 u16 max_contig; 2374 2375 mutex_lock(&mcam->lock); 2376 2377 /* Check if there are any free entries */ 2378 if (!mcam->bmap_fcnt) { 2379 mutex_unlock(&mcam->lock); 2380 return NPC_MCAM_ALLOC_FAILED; 2381 } 2382 2383 /* MCAM entries are divided into high priority, middle and 2384 * low priority zones. Idea is to not allocate top and lower 2385 * most entries as much as possible, this is to increase 2386 * probability of honouring priority allocation requests. 2387 * 2388 * Two bitmaps are used for mcam entry management, 2389 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2390 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2391 * 2392 * Reverse bitmap is used to allocate entries 2393 * - when a higher priority entry is requested 2394 * - when available free entries are less. 2395 * Lower priority ones out of avaialble free entries are always 2396 * chosen when 'high vs low' question arises. 2397 */ 2398 2399 /* Get the search range for priority allocation request */ 2400 if (req->priority) { 2401 npc_get_mcam_search_range_priority(mcam, req, 2402 &start, &end, &reverse); 2403 goto alloc; 2404 } 2405 2406 /* For a VF base MCAM match rule is set by its PF. And all the 2407 * further MCAM rules installed by VF on its own are 2408 * concatenated with the base rule set by its PF. Hence PF entries 2409 * should be at lower priority compared to VF entries. Otherwise 2410 * base rule is hit always and rules installed by VF will be of 2411 * no use. Hence if the request is from PF and NOT a priority 2412 * allocation request then allocate low priority entries. 2413 */ 2414 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 2415 goto lprio_alloc; 2416 2417 /* Find out the search range for non-priority allocation request 2418 * 2419 * Get MCAM free entry count in middle zone. 2420 */ 2421 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2422 mcam->lprio_start, 2423 mcam->bmap_entries); 2424 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2425 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2426 2427 /* Check if request can be accomodated in the middle zone */ 2428 if (fcnt > req->count) { 2429 start = mcam->hprio_end; 2430 end = mcam->lprio_start; 2431 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2432 /* Expand search zone from half of hprio zone to 2433 * half of lprio zone. 2434 */ 2435 start = mcam->hprio_end / 2; 2436 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2437 reverse = true; 2438 } else { 2439 /* Not enough free entries, search all entries in reverse, 2440 * so that low priority ones will get used up. 2441 */ 2442 lprio_alloc: 2443 reverse = true; 2444 start = 0; 2445 end = mcam->bmap_entries; 2446 } 2447 2448 alloc: 2449 if (reverse) { 2450 bmap = mcam->bmap_reverse; 2451 start = mcam->bmap_entries - start; 2452 end = mcam->bmap_entries - end; 2453 swap(start, end); 2454 } else { 2455 bmap = mcam->bmap; 2456 } 2457 2458 if (req->contig) { 2459 /* Allocate requested number of contiguous entries, if 2460 * unsuccessful find max contiguous entries available. 2461 */ 2462 index = npc_mcam_find_zero_area(bmap, end, start, 2463 req->count, &max_contig); 2464 rsp->count = max_contig; 2465 if (reverse) 2466 rsp->entry = mcam->bmap_entries - index - max_contig; 2467 else 2468 rsp->entry = index; 2469 } else { 2470 /* Allocate requested number of non-contiguous entries, 2471 * if unsuccessful allocate as many as possible. 2472 */ 2473 rsp->count = 0; 2474 next_start = start; 2475 for (entry = 0; entry < req->count; entry++) { 2476 index = find_next_zero_bit(bmap, end, next_start); 2477 if (index >= end) 2478 break; 2479 2480 next_start = start + (index - start) + 1; 2481 2482 /* Save the entry's index */ 2483 if (reverse) 2484 index = mcam->bmap_entries - index - 1; 2485 entry_list[entry] = index; 2486 rsp->count++; 2487 } 2488 } 2489 2490 /* If allocating requested no of entries is unsucessful, 2491 * expand the search range to full bitmap length and retry. 2492 */ 2493 if (!req->priority && (rsp->count < req->count) && 2494 ((end - start) != mcam->bmap_entries)) { 2495 reverse = true; 2496 start = 0; 2497 end = mcam->bmap_entries; 2498 goto alloc; 2499 } 2500 2501 /* For priority entry allocation requests, if allocation is 2502 * failed then expand search to max possible range and retry. 2503 */ 2504 if (req->priority && rsp->count < req->count) { 2505 if (req->priority == NPC_MCAM_LOWER_PRIO && 2506 (start != (req->ref_entry + 1))) { 2507 start = req->ref_entry + 1; 2508 end = mcam->bmap_entries; 2509 reverse = false; 2510 goto alloc; 2511 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2512 ((end - start) != req->ref_entry)) { 2513 start = 0; 2514 end = req->ref_entry; 2515 reverse = true; 2516 goto alloc; 2517 } 2518 } 2519 2520 /* Copy MCAM entry indices into mbox response entry_list. 2521 * Requester always expects indices in ascending order, so 2522 * so reverse the list if reverse bitmap is used for allocation. 2523 */ 2524 if (!req->contig && rsp->count) { 2525 index = 0; 2526 for (entry = rsp->count - 1; entry >= 0; entry--) { 2527 if (reverse) 2528 rsp->entry_list[index++] = entry_list[entry]; 2529 else 2530 rsp->entry_list[entry] = entry_list[entry]; 2531 } 2532 } 2533 2534 /* Mark the allocated entries as used and set nixlf mapping */ 2535 for (entry = 0; entry < rsp->count; entry++) { 2536 index = req->contig ? 2537 (rsp->entry + entry) : rsp->entry_list[entry]; 2538 npc_mcam_set_bit(mcam, index); 2539 mcam->entry2pfvf_map[index] = pcifunc; 2540 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2541 } 2542 2543 /* Update available free count in mbox response */ 2544 rsp->free_count = mcam->bmap_fcnt; 2545 2546 mutex_unlock(&mcam->lock); 2547 return 0; 2548 } 2549 2550 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2551 struct npc_mcam_alloc_entry_req *req, 2552 struct npc_mcam_alloc_entry_rsp *rsp) 2553 { 2554 struct npc_mcam *mcam = &rvu->hw->mcam; 2555 u16 pcifunc = req->hdr.pcifunc; 2556 int blkaddr; 2557 2558 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2559 if (blkaddr < 0) 2560 return NPC_MCAM_INVALID_REQ; 2561 2562 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2563 rsp->free_count = 0; 2564 2565 /* Check if ref_entry is within range */ 2566 if (req->priority && req->ref_entry >= mcam->bmap_entries) { 2567 dev_err(rvu->dev, "%s: reference entry %d is out of range\n", 2568 __func__, req->ref_entry); 2569 return NPC_MCAM_INVALID_REQ; 2570 } 2571 2572 /* ref_entry can't be '0' if requested priority is high. 2573 * Can't be last entry if requested priority is low. 2574 */ 2575 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2576 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2577 req->priority == NPC_MCAM_LOWER_PRIO)) 2578 return NPC_MCAM_INVALID_REQ; 2579 2580 /* Since list of allocated indices needs to be sent to requester, 2581 * max number of non-contiguous entries per mbox msg is limited. 2582 */ 2583 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 2584 dev_err(rvu->dev, 2585 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 2586 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 2587 return NPC_MCAM_INVALID_REQ; 2588 } 2589 2590 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2591 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2592 return NPC_MCAM_ALLOC_DENIED; 2593 2594 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2595 } 2596 2597 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2598 struct npc_mcam_free_entry_req *req, 2599 struct msg_rsp *rsp) 2600 { 2601 struct npc_mcam *mcam = &rvu->hw->mcam; 2602 u16 pcifunc = req->hdr.pcifunc; 2603 int blkaddr, rc = 0; 2604 u16 cntr; 2605 2606 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2607 if (blkaddr < 0) 2608 return NPC_MCAM_INVALID_REQ; 2609 2610 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2611 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2612 return NPC_MCAM_INVALID_REQ; 2613 2614 mutex_lock(&mcam->lock); 2615 2616 if (req->all) 2617 goto free_all; 2618 2619 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2620 if (rc) 2621 goto exit; 2622 2623 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 2624 mcam->entry2target_pffunc[req->entry] = 0x0; 2625 npc_mcam_clear_bit(mcam, req->entry); 2626 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2627 2628 /* Update entry2counter mapping */ 2629 cntr = mcam->entry2cntr_map[req->entry]; 2630 if (cntr != NPC_MCAM_INVALID_MAP) 2631 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2632 req->entry, cntr); 2633 2634 goto exit; 2635 2636 free_all: 2637 /* Free up all entries allocated to requesting PFFUNC */ 2638 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2639 exit: 2640 mutex_unlock(&mcam->lock); 2641 return rc; 2642 } 2643 2644 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2645 struct npc_mcam_read_entry_req *req, 2646 struct npc_mcam_read_entry_rsp *rsp) 2647 { 2648 struct npc_mcam *mcam = &rvu->hw->mcam; 2649 u16 pcifunc = req->hdr.pcifunc; 2650 int blkaddr, rc; 2651 2652 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2653 if (blkaddr < 0) 2654 return NPC_MCAM_INVALID_REQ; 2655 2656 mutex_lock(&mcam->lock); 2657 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2658 if (!rc) { 2659 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2660 &rsp->entry_data, 2661 &rsp->intf, &rsp->enable); 2662 } 2663 2664 mutex_unlock(&mcam->lock); 2665 return rc; 2666 } 2667 2668 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2669 struct npc_mcam_write_entry_req *req, 2670 struct msg_rsp *rsp) 2671 { 2672 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2673 struct npc_mcam *mcam = &rvu->hw->mcam; 2674 u16 pcifunc = req->hdr.pcifunc; 2675 int blkaddr, rc; 2676 u8 nix_intf; 2677 2678 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2679 if (blkaddr < 0) 2680 return NPC_MCAM_INVALID_REQ; 2681 2682 mutex_lock(&mcam->lock); 2683 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2684 if (rc) 2685 goto exit; 2686 2687 if (req->set_cntr && 2688 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2689 rc = NPC_MCAM_INVALID_REQ; 2690 goto exit; 2691 } 2692 2693 if (!is_npc_interface_valid(rvu, req->intf)) { 2694 rc = NPC_MCAM_INVALID_REQ; 2695 goto exit; 2696 } 2697 2698 if (is_npc_intf_tx(req->intf)) 2699 nix_intf = pfvf->nix_tx_intf; 2700 else 2701 nix_intf = pfvf->nix_rx_intf; 2702 2703 if (!is_pffunc_af(pcifunc) && 2704 npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, pcifunc)) { 2705 rc = NPC_MCAM_INVALID_REQ; 2706 goto exit; 2707 } 2708 2709 /* For AF installed rules, the nix_intf should be set to target NIX */ 2710 if (is_pffunc_af(req->hdr.pcifunc)) 2711 nix_intf = req->intf; 2712 2713 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2714 &req->entry_data, req->enable_entry); 2715 2716 if (req->set_cntr) 2717 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2718 req->entry, req->cntr); 2719 2720 rc = 0; 2721 exit: 2722 mutex_unlock(&mcam->lock); 2723 return rc; 2724 } 2725 2726 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2727 struct npc_mcam_ena_dis_entry_req *req, 2728 struct msg_rsp *rsp) 2729 { 2730 struct npc_mcam *mcam = &rvu->hw->mcam; 2731 u16 pcifunc = req->hdr.pcifunc; 2732 int blkaddr, rc; 2733 2734 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2735 if (blkaddr < 0) 2736 return NPC_MCAM_INVALID_REQ; 2737 2738 mutex_lock(&mcam->lock); 2739 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2740 mutex_unlock(&mcam->lock); 2741 if (rc) 2742 return rc; 2743 2744 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2745 2746 return 0; 2747 } 2748 2749 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2750 struct npc_mcam_ena_dis_entry_req *req, 2751 struct msg_rsp *rsp) 2752 { 2753 struct npc_mcam *mcam = &rvu->hw->mcam; 2754 u16 pcifunc = req->hdr.pcifunc; 2755 int blkaddr, rc; 2756 2757 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2758 if (blkaddr < 0) 2759 return NPC_MCAM_INVALID_REQ; 2760 2761 mutex_lock(&mcam->lock); 2762 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2763 mutex_unlock(&mcam->lock); 2764 if (rc) 2765 return rc; 2766 2767 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2768 2769 return 0; 2770 } 2771 2772 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2773 struct npc_mcam_shift_entry_req *req, 2774 struct npc_mcam_shift_entry_rsp *rsp) 2775 { 2776 struct npc_mcam *mcam = &rvu->hw->mcam; 2777 u16 pcifunc = req->hdr.pcifunc; 2778 u16 old_entry, new_entry; 2779 int blkaddr, rc = 0; 2780 u16 index, cntr; 2781 2782 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2783 if (blkaddr < 0) 2784 return NPC_MCAM_INVALID_REQ; 2785 2786 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2787 return NPC_MCAM_INVALID_REQ; 2788 2789 mutex_lock(&mcam->lock); 2790 for (index = 0; index < req->shift_count; index++) { 2791 old_entry = req->curr_entry[index]; 2792 new_entry = req->new_entry[index]; 2793 2794 /* Check if both old and new entries are valid and 2795 * does belong to this PFFUNC or not. 2796 */ 2797 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2798 if (rc) 2799 break; 2800 2801 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2802 if (rc) 2803 break; 2804 2805 /* new_entry should not have a counter mapped */ 2806 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2807 rc = NPC_MCAM_PERM_DENIED; 2808 break; 2809 } 2810 2811 /* Disable the new_entry */ 2812 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2813 2814 /* Copy rule from old entry to new entry */ 2815 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2816 2817 /* Copy counter mapping, if any */ 2818 cntr = mcam->entry2cntr_map[old_entry]; 2819 if (cntr != NPC_MCAM_INVALID_MAP) { 2820 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2821 old_entry, cntr); 2822 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2823 new_entry, cntr); 2824 } 2825 2826 /* Enable new_entry and disable old_entry */ 2827 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2828 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2829 } 2830 2831 /* If shift has failed then report the failed index */ 2832 if (index != req->shift_count) { 2833 rc = NPC_MCAM_PERM_DENIED; 2834 rsp->failed_entry_idx = index; 2835 } 2836 2837 mutex_unlock(&mcam->lock); 2838 return rc; 2839 } 2840 2841 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2842 struct npc_mcam_alloc_counter_req *req, 2843 struct npc_mcam_alloc_counter_rsp *rsp) 2844 { 2845 struct npc_mcam *mcam = &rvu->hw->mcam; 2846 u16 pcifunc = req->hdr.pcifunc; 2847 u16 max_contig, cntr; 2848 int blkaddr, index; 2849 2850 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2851 if (blkaddr < 0) 2852 return NPC_MCAM_INVALID_REQ; 2853 2854 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2855 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2856 return NPC_MCAM_INVALID_REQ; 2857 2858 /* Since list of allocated counter IDs needs to be sent to requester, 2859 * max number of non-contiguous counters per mbox msg is limited. 2860 */ 2861 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2862 return NPC_MCAM_INVALID_REQ; 2863 2864 mutex_lock(&mcam->lock); 2865 2866 /* Check if unused counters are available or not */ 2867 if (!rvu_rsrc_free_count(&mcam->counters)) { 2868 mutex_unlock(&mcam->lock); 2869 return NPC_MCAM_ALLOC_FAILED; 2870 } 2871 2872 rsp->count = 0; 2873 2874 if (req->contig) { 2875 /* Allocate requested number of contiguous counters, if 2876 * unsuccessful find max contiguous entries available. 2877 */ 2878 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2879 mcam->counters.max, 0, 2880 req->count, &max_contig); 2881 rsp->count = max_contig; 2882 rsp->cntr = index; 2883 for (cntr = index; cntr < (index + max_contig); cntr++) { 2884 __set_bit(cntr, mcam->counters.bmap); 2885 mcam->cntr2pfvf_map[cntr] = pcifunc; 2886 } 2887 } else { 2888 /* Allocate requested number of non-contiguous counters, 2889 * if unsuccessful allocate as many as possible. 2890 */ 2891 for (cntr = 0; cntr < req->count; cntr++) { 2892 index = rvu_alloc_rsrc(&mcam->counters); 2893 if (index < 0) 2894 break; 2895 rsp->cntr_list[cntr] = index; 2896 rsp->count++; 2897 mcam->cntr2pfvf_map[index] = pcifunc; 2898 } 2899 } 2900 2901 mutex_unlock(&mcam->lock); 2902 return 0; 2903 } 2904 2905 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2906 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2907 { 2908 struct npc_mcam *mcam = &rvu->hw->mcam; 2909 u16 index, entry = 0; 2910 int blkaddr, err; 2911 2912 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2913 if (blkaddr < 0) 2914 return NPC_MCAM_INVALID_REQ; 2915 2916 mutex_lock(&mcam->lock); 2917 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2918 if (err) { 2919 mutex_unlock(&mcam->lock); 2920 return err; 2921 } 2922 2923 /* Mark counter as free/unused */ 2924 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2925 rvu_free_rsrc(&mcam->counters, req->cntr); 2926 2927 /* Disable all MCAM entry's stats which are using this counter */ 2928 while (entry < mcam->bmap_entries) { 2929 if (!mcam->cntr_refcnt[req->cntr]) 2930 break; 2931 2932 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2933 if (index >= mcam->bmap_entries) 2934 break; 2935 entry = index + 1; 2936 if (mcam->entry2cntr_map[index] != req->cntr) 2937 continue; 2938 2939 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2940 index, req->cntr); 2941 } 2942 2943 mutex_unlock(&mcam->lock); 2944 return 0; 2945 } 2946 2947 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2948 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2949 { 2950 struct npc_mcam *mcam = &rvu->hw->mcam; 2951 u16 index, entry = 0; 2952 int blkaddr, rc; 2953 2954 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2955 if (blkaddr < 0) 2956 return NPC_MCAM_INVALID_REQ; 2957 2958 mutex_lock(&mcam->lock); 2959 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2960 if (rc) 2961 goto exit; 2962 2963 /* Unmap the MCAM entry and counter */ 2964 if (!req->all) { 2965 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2966 if (rc) 2967 goto exit; 2968 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2969 req->entry, req->cntr); 2970 goto exit; 2971 } 2972 2973 /* Disable all MCAM entry's stats which are using this counter */ 2974 while (entry < mcam->bmap_entries) { 2975 if (!mcam->cntr_refcnt[req->cntr]) 2976 break; 2977 2978 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2979 if (index >= mcam->bmap_entries) 2980 break; 2981 entry = index + 1; 2982 2983 if (mcam->entry2cntr_map[index] != req->cntr) 2984 continue; 2985 2986 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2987 index, req->cntr); 2988 } 2989 exit: 2990 mutex_unlock(&mcam->lock); 2991 return rc; 2992 } 2993 2994 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2995 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2996 { 2997 struct npc_mcam *mcam = &rvu->hw->mcam; 2998 int blkaddr, err; 2999 3000 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3001 if (blkaddr < 0) 3002 return NPC_MCAM_INVALID_REQ; 3003 3004 mutex_lock(&mcam->lock); 3005 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3006 mutex_unlock(&mcam->lock); 3007 if (err) 3008 return err; 3009 3010 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3011 3012 return 0; 3013 } 3014 3015 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 3016 struct npc_mcam_oper_counter_req *req, 3017 struct npc_mcam_oper_counter_rsp *rsp) 3018 { 3019 struct npc_mcam *mcam = &rvu->hw->mcam; 3020 int blkaddr, err; 3021 3022 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3023 if (blkaddr < 0) 3024 return NPC_MCAM_INVALID_REQ; 3025 3026 mutex_lock(&mcam->lock); 3027 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3028 mutex_unlock(&mcam->lock); 3029 if (err) 3030 return err; 3031 3032 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3033 rsp->stat &= BIT_ULL(48) - 1; 3034 3035 return 0; 3036 } 3037 3038 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3039 struct npc_mcam_alloc_and_write_entry_req *req, 3040 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3041 { 3042 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3043 struct npc_mcam_alloc_counter_req cntr_req; 3044 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3045 struct npc_mcam_alloc_entry_req entry_req; 3046 struct npc_mcam_alloc_entry_rsp entry_rsp; 3047 struct npc_mcam *mcam = &rvu->hw->mcam; 3048 u16 entry = NPC_MCAM_ENTRY_INVALID; 3049 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3050 int blkaddr, rc; 3051 u8 nix_intf; 3052 3053 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3054 if (blkaddr < 0) 3055 return NPC_MCAM_INVALID_REQ; 3056 3057 if (!is_npc_interface_valid(rvu, req->intf)) 3058 return NPC_MCAM_INVALID_REQ; 3059 3060 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 3061 req->hdr.pcifunc)) 3062 return NPC_MCAM_INVALID_REQ; 3063 3064 /* Try to allocate a MCAM entry */ 3065 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3066 entry_req.contig = true; 3067 entry_req.priority = req->priority; 3068 entry_req.ref_entry = req->ref_entry; 3069 entry_req.count = 1; 3070 3071 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3072 &entry_req, &entry_rsp); 3073 if (rc) 3074 return rc; 3075 3076 if (!entry_rsp.count) 3077 return NPC_MCAM_ALLOC_FAILED; 3078 3079 entry = entry_rsp.entry; 3080 3081 if (!req->alloc_cntr) 3082 goto write_entry; 3083 3084 /* Now allocate counter */ 3085 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3086 cntr_req.contig = true; 3087 cntr_req.count = 1; 3088 3089 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3090 if (rc) { 3091 /* Free allocated MCAM entry */ 3092 mutex_lock(&mcam->lock); 3093 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 3094 npc_mcam_clear_bit(mcam, entry); 3095 mutex_unlock(&mcam->lock); 3096 return rc; 3097 } 3098 3099 cntr = cntr_rsp.cntr; 3100 3101 write_entry: 3102 mutex_lock(&mcam->lock); 3103 3104 if (is_npc_intf_tx(req->intf)) 3105 nix_intf = pfvf->nix_tx_intf; 3106 else 3107 nix_intf = pfvf->nix_rx_intf; 3108 3109 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3110 &req->entry_data, req->enable_entry); 3111 3112 if (req->alloc_cntr) 3113 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3114 mutex_unlock(&mcam->lock); 3115 3116 rsp->entry = entry; 3117 rsp->cntr = cntr; 3118 3119 return 0; 3120 } 3121 3122 #define GET_KEX_CFG(intf) \ 3123 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3124 3125 #define GET_KEX_FLAGS(ld) \ 3126 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3127 3128 #define GET_KEX_LD(intf, lid, lt, ld) \ 3129 rvu_read64(rvu, BLKADDR_NPC, \ 3130 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3131 3132 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3133 rvu_read64(rvu, BLKADDR_NPC, \ 3134 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3135 3136 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3137 struct npc_get_kex_cfg_rsp *rsp) 3138 { 3139 int lid, lt, ld, fl; 3140 3141 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3142 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3143 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3144 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3145 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3146 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3147 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3148 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3149 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3150 } 3151 } 3152 } 3153 for (ld = 0; ld < NPC_MAX_LD; ld++) 3154 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3155 3156 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3157 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3158 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3159 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3160 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3161 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3162 } 3163 } 3164 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3165 return 0; 3166 } 3167 3168 static int 3169 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 3170 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 3171 { 3172 struct npc_kpu_action0 *act0; 3173 u8 shift_count = 0; 3174 int blkaddr; 3175 u64 val; 3176 3177 if (!var_len_off_mask) 3178 return -EINVAL; 3179 3180 if (var_len_off_mask != 0xff) { 3181 if (shift_dir) 3182 shift_count = __ffs(var_len_off_mask); 3183 else 3184 shift_count = (8 - __fls(var_len_off_mask)); 3185 } 3186 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 3187 if (blkaddr < 0) { 3188 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 3189 return -EINVAL; 3190 } 3191 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 3192 act0 = (struct npc_kpu_action0 *)&val; 3193 act0->var_len_shift = shift_count; 3194 act0->var_len_right = shift_dir; 3195 act0->var_len_mask = var_len_off_mask; 3196 act0->var_len_offset = var_len_off; 3197 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 3198 return 0; 3199 } 3200 3201 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 3202 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 3203 u8 shift_dir) 3204 3205 { 3206 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 3207 int blkaddr, nixlf, rc, intf_mode; 3208 int pf = rvu_get_pf(pcifunc); 3209 u64 rxpkind, txpkind; 3210 u8 cgx_id, lmac_id; 3211 3212 /* use default pkind to disable edsa/higig */ 3213 rxpkind = rvu_npc_get_pkind(rvu, pf); 3214 txpkind = NPC_TX_DEF_PKIND; 3215 intf_mode = NPC_INTF_MODE_DEF; 3216 3217 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 3218 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 3219 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 3220 var_len_off, 3221 var_len_off_mask, 3222 shift_dir); 3223 if (rc) 3224 return rc; 3225 } 3226 rxpkind = pkind; 3227 txpkind = pkind; 3228 } 3229 3230 if (dir & PKIND_RX) { 3231 /* rx pkind set req valid only for cgx mapped PFs */ 3232 if (!is_cgx_config_permitted(rvu, pcifunc)) 3233 return 0; 3234 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3235 3236 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 3237 rxpkind); 3238 if (rc) 3239 return rc; 3240 } 3241 3242 if (dir & PKIND_TX) { 3243 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 3244 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3245 if (rc) 3246 return rc; 3247 3248 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 3249 txpkind); 3250 } 3251 3252 pfvf->intf_mode = intf_mode; 3253 return 0; 3254 } 3255 3256 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 3257 struct msg_rsp *rsp) 3258 { 3259 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 3260 req->dir, req->pkind, req->var_len_off, 3261 req->var_len_off_mask, req->shift_dir); 3262 } 3263 3264 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3265 struct msg_req *req, 3266 struct npc_mcam_read_base_rule_rsp *rsp) 3267 { 3268 struct npc_mcam *mcam = &rvu->hw->mcam; 3269 int index, blkaddr, nixlf, rc = 0; 3270 u16 pcifunc = req->hdr.pcifunc; 3271 struct rvu_pfvf *pfvf; 3272 u8 intf, enable; 3273 3274 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3275 if (blkaddr < 0) 3276 return NPC_MCAM_INVALID_REQ; 3277 3278 /* Return the channel number in case of PF */ 3279 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3280 pfvf = rvu_get_pfvf(rvu, pcifunc); 3281 rsp->entry.kw[0] = pfvf->rx_chan_base; 3282 rsp->entry.kw_mask[0] = 0xFFFULL; 3283 goto out; 3284 } 3285 3286 /* Find the pkt steering rule installed by PF to this VF */ 3287 mutex_lock(&mcam->lock); 3288 for (index = 0; index < mcam->bmap_entries; index++) { 3289 if (mcam->entry2target_pffunc[index] == pcifunc) 3290 goto read_entry; 3291 } 3292 3293 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3294 if (rc < 0) { 3295 mutex_unlock(&mcam->lock); 3296 goto out; 3297 } 3298 /* Read the default ucast entry if there is no pkt steering rule */ 3299 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3300 NIXLF_UCAST_ENTRY); 3301 read_entry: 3302 /* Read the mcam entry */ 3303 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3304 &enable); 3305 mutex_unlock(&mcam->lock); 3306 out: 3307 return rc; 3308 } 3309 3310 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3311 struct npc_mcam_get_stats_req *req, 3312 struct npc_mcam_get_stats_rsp *rsp) 3313 { 3314 struct npc_mcam *mcam = &rvu->hw->mcam; 3315 u16 index, cntr; 3316 int blkaddr; 3317 u64 regval; 3318 u32 bank; 3319 3320 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3321 if (blkaddr < 0) 3322 return NPC_MCAM_INVALID_REQ; 3323 3324 mutex_lock(&mcam->lock); 3325 3326 index = req->entry & (mcam->banksize - 1); 3327 bank = npc_get_bank(mcam, req->entry); 3328 3329 /* read MCAM entry STAT_ACT register */ 3330 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3331 3332 if (!(regval & rvu->hw->npc_stat_ena)) { 3333 rsp->stat_ena = 0; 3334 mutex_unlock(&mcam->lock); 3335 return 0; 3336 } 3337 3338 cntr = regval & 0x1FF; 3339 3340 rsp->stat_ena = 1; 3341 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3342 rsp->stat &= BIT_ULL(48) - 1; 3343 3344 mutex_unlock(&mcam->lock); 3345 3346 return 0; 3347 } 3348