1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 12 #include "rvu_struct.h" 13 #include "rvu_reg.h" 14 #include "rvu.h" 15 #include "npc.h" 16 #include "cgx.h" 17 #include "npc_profile.h" 18 19 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 20 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 21 22 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 23 #define NPC_HW_TSTAMP_OFFSET 8ULL 24 #define NPC_KEX_CHAN_MASK 0xFFFULL 25 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 26 27 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 28 29 static const char def_pfl_name[] = "default"; 30 31 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 32 int blkaddr, u16 pcifunc); 33 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 34 u16 pcifunc); 35 36 bool is_npc_intf_tx(u8 intf) 37 { 38 return !!(intf & 0x1); 39 } 40 41 bool is_npc_intf_rx(u8 intf) 42 { 43 return !(intf & 0x1); 44 } 45 46 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 47 { 48 struct rvu_hwinfo *hw = rvu->hw; 49 50 return intf < hw->npc_intfs; 51 } 52 53 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 54 { 55 /* Due to a HW issue in these silicon versions, parse nibble enable 56 * configuration has to be identical for both Rx and Tx interfaces. 57 */ 58 if (is_rvu_96xx_B0(rvu)) 59 return nibble_ena; 60 return 0; 61 } 62 63 static int npc_mcam_verify_pf_func(struct rvu *rvu, 64 struct mcam_entry *entry_data, u8 intf, 65 u16 pcifunc) 66 { 67 u16 pf_func, pf_func_mask; 68 69 if (is_npc_intf_rx(intf)) 70 return 0; 71 72 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 73 NPC_KEX_PF_FUNC_MASK; 74 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 75 76 pf_func = be16_to_cpu((__force __be16)pf_func); 77 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 78 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 79 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 80 return -EINVAL; 81 82 return 0; 83 } 84 85 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 86 { 87 int blkaddr; 88 u64 val = 0; 89 90 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 91 if (blkaddr < 0) 92 return; 93 94 /* Config CPI base for the PKIND */ 95 val = pkind | 1ULL << 62; 96 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 97 } 98 99 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 100 { 101 struct npc_pkind *pkind = &rvu->hw->pkind; 102 u32 map; 103 int i; 104 105 for (i = 0; i < pkind->rsrc.max; i++) { 106 map = pkind->pfchan_map[i]; 107 if (((map >> 16) & 0x3F) == pf) 108 return i; 109 } 110 return -1; 111 } 112 113 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 114 115 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 116 { 117 int pkind, blkaddr; 118 u64 val; 119 120 pkind = rvu_npc_get_pkind(rvu, pf); 121 if (pkind < 0) { 122 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 123 return -EINVAL; 124 } 125 126 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 127 if (blkaddr < 0) { 128 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 129 return -EINVAL; 130 } 131 132 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 133 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 134 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 135 if (enable) 136 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 137 NPC_HW_TSTAMP_OFFSET); 138 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 139 140 return 0; 141 } 142 143 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 144 int nixlf) 145 { 146 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 147 struct rvu *rvu = hw->rvu; 148 int blkaddr = 0, max = 0; 149 struct rvu_block *block; 150 struct rvu_pfvf *pfvf; 151 152 pfvf = rvu_get_pfvf(rvu, pcifunc); 153 /* Given a PF/VF and NIX LF number calculate the unicast mcam 154 * entry index based on the NIX block assigned to the PF/VF. 155 */ 156 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 157 while (blkaddr) { 158 if (pfvf->nix_blkaddr == blkaddr) 159 break; 160 block = &rvu->hw->block[blkaddr]; 161 max += block->lf.max; 162 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 163 } 164 165 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 166 } 167 168 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 169 u16 pcifunc, int nixlf, int type) 170 { 171 int pf = rvu_get_pf(pcifunc); 172 int index; 173 174 /* Check if this is for a PF */ 175 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 176 /* Reserved entries exclude PF0 */ 177 pf--; 178 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 179 /* Broadcast address matching entry should be first so 180 * that the packet can be replicated to all VFs. 181 */ 182 if (type == NIXLF_BCAST_ENTRY) 183 return index; 184 else if (type == NIXLF_ALLMULTI_ENTRY) 185 return index + 1; 186 else if (type == NIXLF_PROMISC_ENTRY) 187 return index + 2; 188 } 189 190 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 191 } 192 193 int npc_get_bank(struct npc_mcam *mcam, int index) 194 { 195 int bank = index / mcam->banksize; 196 197 /* 0,1 & 2,3 banks are combined for this keysize */ 198 if (mcam->keysize == NPC_MCAM_KEY_X2) 199 return bank ? 2 : 0; 200 201 return bank; 202 } 203 204 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 205 int blkaddr, int index) 206 { 207 int bank = npc_get_bank(mcam, index); 208 u64 cfg; 209 210 index &= (mcam->banksize - 1); 211 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 212 return (cfg & 1); 213 } 214 215 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 216 int blkaddr, int index, bool enable) 217 { 218 int bank = npc_get_bank(mcam, index); 219 int actbank = bank; 220 221 index &= (mcam->banksize - 1); 222 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 223 rvu_write64(rvu, blkaddr, 224 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 225 enable ? 1 : 0); 226 } 227 } 228 229 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 230 int blkaddr, int index) 231 { 232 int bank = npc_get_bank(mcam, index); 233 int actbank = bank; 234 235 index &= (mcam->banksize - 1); 236 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 237 rvu_write64(rvu, blkaddr, 238 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 239 rvu_write64(rvu, blkaddr, 240 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 241 242 rvu_write64(rvu, blkaddr, 243 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 244 rvu_write64(rvu, blkaddr, 245 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 246 247 rvu_write64(rvu, blkaddr, 248 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 249 rvu_write64(rvu, blkaddr, 250 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 251 } 252 } 253 254 static void npc_get_keyword(struct mcam_entry *entry, int idx, 255 u64 *cam0, u64 *cam1) 256 { 257 u64 kw_mask = 0x00; 258 259 #define CAM_MASK(n) (BIT_ULL(n) - 1) 260 261 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 262 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 263 * 264 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 265 */ 266 switch (idx) { 267 case 0: 268 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 269 *cam1 = entry->kw[0]; 270 kw_mask = entry->kw_mask[0]; 271 break; 272 case 1: 273 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 274 *cam1 = entry->kw[1] & CAM_MASK(48); 275 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 276 break; 277 case 2: 278 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 279 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 280 */ 281 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 282 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 283 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 284 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 285 break; 286 case 3: 287 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 288 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 289 */ 290 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 291 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 292 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 293 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 294 break; 295 case 4: 296 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 297 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 298 */ 299 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 300 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 301 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 302 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 303 break; 304 case 5: 305 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 306 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 307 */ 308 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 309 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 310 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 311 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 312 break; 313 case 6: 314 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 315 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 316 */ 317 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 318 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 319 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 320 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 321 break; 322 case 7: 323 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 324 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 325 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 326 break; 327 } 328 329 *cam1 &= kw_mask; 330 *cam0 = ~*cam1 & kw_mask; 331 } 332 333 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 334 u64 cam0, u64 cam1) 335 { 336 /* Similar to npc_get_keyword, but fills mcam_entry structure from 337 * CAM registers. 338 */ 339 switch (idx) { 340 case 0: 341 entry->kw[0] = cam1; 342 entry->kw_mask[0] = cam1 ^ cam0; 343 break; 344 case 1: 345 entry->kw[1] = cam1; 346 entry->kw_mask[1] = cam1 ^ cam0; 347 break; 348 case 2: 349 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 350 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 351 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 352 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 353 break; 354 case 3: 355 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 356 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 357 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 358 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 359 break; 360 case 4: 361 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 362 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 363 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 364 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 365 break; 366 case 5: 367 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 368 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 369 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 370 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 371 break; 372 case 6: 373 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 374 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 375 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 376 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 377 break; 378 case 7: 379 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 380 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 381 break; 382 } 383 } 384 385 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 386 int blkaddr, u16 pf_func) 387 { 388 int bank, nixlf, index; 389 390 /* get ucast entry rule entry index */ 391 nix_get_nixlf(rvu, pf_func, &nixlf, NULL); 392 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 393 NIXLF_UCAST_ENTRY); 394 bank = npc_get_bank(mcam, index); 395 index &= (mcam->banksize - 1); 396 397 return rvu_read64(rvu, blkaddr, 398 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 399 } 400 401 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 402 int blkaddr, int index, struct mcam_entry *entry, 403 bool *enable) 404 { 405 struct rvu_npc_mcam_rule *rule; 406 u16 owner, target_func; 407 struct rvu_pfvf *pfvf; 408 u64 rx_action; 409 410 owner = mcam->entry2pfvf_map[index]; 411 target_func = (entry->action >> 4) & 0xffff; 412 /* do nothing when target is LBK/PF or owner is not PF */ 413 if (is_pffunc_af(owner) || is_afvf(target_func) || 414 (owner & RVU_PFVF_FUNC_MASK) || 415 !(target_func & RVU_PFVF_FUNC_MASK)) 416 return; 417 418 /* save entry2target_pffunc */ 419 pfvf = rvu_get_pfvf(rvu, target_func); 420 mcam->entry2target_pffunc[index] = target_func; 421 422 /* don't enable rule when nixlf not attached or initialized */ 423 if (!(is_nixlf_attached(rvu, target_func) && 424 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 425 *enable = false; 426 427 /* fix up not needed for the rules added by user(ntuple filters) */ 428 list_for_each_entry(rule, &mcam->mcam_rules, list) { 429 if (rule->entry == index) 430 return; 431 } 432 433 /* copy VF default entry action to the VF mcam entry */ 434 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 435 target_func); 436 if (rx_action) 437 entry->action = rx_action; 438 } 439 440 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 441 int blkaddr, int index, u8 intf, 442 struct mcam_entry *entry, bool enable) 443 { 444 int bank = npc_get_bank(mcam, index); 445 int kw = 0, actbank, actindex; 446 u8 tx_intf_mask = ~intf & 0x3; 447 u8 tx_intf = intf; 448 u64 cam0, cam1; 449 450 actbank = bank; /* Save bank id, to set action later on */ 451 actindex = index; 452 index &= (mcam->banksize - 1); 453 454 /* Disable before mcam entry update */ 455 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 456 457 /* Clear mcam entry to avoid writes being suppressed by NPC */ 458 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 459 460 /* CAM1 takes the comparison value and 461 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 462 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 463 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 464 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 465 */ 466 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 467 /* Interface should be set in all banks */ 468 if (is_npc_intf_tx(intf)) { 469 /* Last bit must be set and rest don't care 470 * for TX interfaces 471 */ 472 tx_intf_mask = 0x1; 473 tx_intf = intf & tx_intf_mask; 474 tx_intf_mask = ~tx_intf & tx_intf_mask; 475 } 476 477 rvu_write64(rvu, blkaddr, 478 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 479 tx_intf); 480 rvu_write64(rvu, blkaddr, 481 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 482 tx_intf_mask); 483 484 /* Set the match key */ 485 npc_get_keyword(entry, kw, &cam0, &cam1); 486 rvu_write64(rvu, blkaddr, 487 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 488 rvu_write64(rvu, blkaddr, 489 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 490 491 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 492 rvu_write64(rvu, blkaddr, 493 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 494 rvu_write64(rvu, blkaddr, 495 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 496 } 497 498 /* PF installing VF rule */ 499 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries) 500 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable); 501 502 /* Set 'action' */ 503 rvu_write64(rvu, blkaddr, 504 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 505 506 /* Set TAG 'action' */ 507 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 508 entry->vtag_action); 509 510 /* Enable the entry */ 511 if (enable) 512 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 513 } 514 515 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 516 int blkaddr, u16 src, 517 struct mcam_entry *entry, u8 *intf, u8 *ena) 518 { 519 int sbank = npc_get_bank(mcam, src); 520 int bank, kw = 0; 521 u64 cam0, cam1; 522 523 src &= (mcam->banksize - 1); 524 bank = sbank; 525 526 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 527 cam1 = rvu_read64(rvu, blkaddr, 528 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 529 cam0 = rvu_read64(rvu, blkaddr, 530 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 531 npc_fill_entryword(entry, kw, cam0, cam1); 532 533 cam1 = rvu_read64(rvu, blkaddr, 534 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 535 cam0 = rvu_read64(rvu, blkaddr, 536 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 537 npc_fill_entryword(entry, kw + 1, cam0, cam1); 538 } 539 540 entry->action = rvu_read64(rvu, blkaddr, 541 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 542 entry->vtag_action = 543 rvu_read64(rvu, blkaddr, 544 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 545 *intf = rvu_read64(rvu, blkaddr, 546 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 547 *ena = rvu_read64(rvu, blkaddr, 548 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 549 } 550 551 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 552 int blkaddr, u16 src, u16 dest) 553 { 554 int dbank = npc_get_bank(mcam, dest); 555 int sbank = npc_get_bank(mcam, src); 556 u64 cfg, sreg, dreg; 557 int bank, i; 558 559 src &= (mcam->banksize - 1); 560 dest &= (mcam->banksize - 1); 561 562 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 563 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 564 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 565 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 566 for (i = 0; i < 6; i++) { 567 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 568 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 569 } 570 } 571 572 /* Copy action */ 573 cfg = rvu_read64(rvu, blkaddr, 574 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 575 rvu_write64(rvu, blkaddr, 576 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 577 578 /* Copy TAG action */ 579 cfg = rvu_read64(rvu, blkaddr, 580 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 581 rvu_write64(rvu, blkaddr, 582 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 583 584 /* Enable or disable */ 585 cfg = rvu_read64(rvu, blkaddr, 586 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 587 rvu_write64(rvu, blkaddr, 588 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 589 } 590 591 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 592 int blkaddr, int index) 593 { 594 int bank = npc_get_bank(mcam, index); 595 596 index &= (mcam->banksize - 1); 597 return rvu_read64(rvu, blkaddr, 598 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 599 } 600 601 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 602 int nixlf, u64 chan, u8 *mac_addr) 603 { 604 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 605 struct npc_install_flow_req req = { 0 }; 606 struct npc_install_flow_rsp rsp = { 0 }; 607 struct npc_mcam *mcam = &rvu->hw->mcam; 608 struct nix_rx_action action; 609 int blkaddr, index; 610 611 /* AF's and SDP VFs work in promiscuous mode */ 612 if (is_afvf(pcifunc) || is_sdp_vf(pcifunc)) 613 return; 614 615 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 616 if (blkaddr < 0) 617 return; 618 619 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 620 nixlf, NIXLF_UCAST_ENTRY); 621 622 /* Don't change the action if entry is already enabled 623 * Otherwise RSS action may get overwritten. 624 */ 625 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 626 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 627 blkaddr, index); 628 } else { 629 *(u64 *)&action = 0x00; 630 action.op = NIX_RX_ACTIONOP_UCAST; 631 action.pf_func = pcifunc; 632 } 633 634 req.default_rule = 1; 635 ether_addr_copy(req.packet.dmac, mac_addr); 636 eth_broadcast_addr((u8 *)&req.mask.dmac); 637 req.features = BIT_ULL(NPC_DMAC); 638 req.channel = chan; 639 req.chan_mask = 0xFFFU; 640 req.intf = pfvf->nix_rx_intf; 641 req.op = action.op; 642 req.hdr.pcifunc = 0; /* AF is requester */ 643 req.vf = action.pf_func; 644 req.index = action.index; 645 req.match_id = action.match_id; 646 req.flow_key_alg = action.flow_key_alg; 647 648 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 649 } 650 651 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 652 int nixlf, u64 chan, u8 chan_cnt) 653 { 654 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 655 struct npc_install_flow_req req = { 0 }; 656 struct npc_install_flow_rsp rsp = { 0 }; 657 struct npc_mcam *mcam = &rvu->hw->mcam; 658 struct rvu_hwinfo *hw = rvu->hw; 659 int blkaddr, ucast_idx, index; 660 struct nix_rx_action action; 661 u64 relaxed_mask; 662 663 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 664 return; 665 666 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 667 if (blkaddr < 0) 668 return; 669 670 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 671 nixlf, NIXLF_PROMISC_ENTRY); 672 673 if (is_cgx_vf(rvu, pcifunc)) 674 index = npc_get_nixlf_mcam_index(mcam, 675 pcifunc & ~RVU_PFVF_FUNC_MASK, 676 nixlf, NIXLF_PROMISC_ENTRY); 677 678 /* If the corresponding PF's ucast action is RSS, 679 * use the same action for promisc also 680 */ 681 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 682 nixlf, NIXLF_UCAST_ENTRY); 683 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 684 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 685 blkaddr, ucast_idx); 686 687 if (action.op != NIX_RX_ACTIONOP_RSS) { 688 *(u64 *)&action = 0x00; 689 action.op = NIX_RX_ACTIONOP_UCAST; 690 } 691 692 /* RX_ACTION set to MCAST for CGX PF's */ 693 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 694 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { 695 *(u64 *)&action = 0x00; 696 action.op = NIX_RX_ACTIONOP_MCAST; 697 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 698 action.index = pfvf->promisc_mce_idx; 699 } 700 701 /* For cn10k the upper two bits of the channel number are 702 * cpt channel number. with masking out these bits in the 703 * mcam entry, same entry used for NIX will allow packets 704 * received from cpt for parsing. 705 */ 706 if (!is_rvu_otx2(rvu)) { 707 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 708 } else { 709 req.chan_mask = 0xFFFU; 710 } 711 712 if (chan_cnt > 1) { 713 if (!is_power_of_2(chan_cnt)) { 714 dev_err(rvu->dev, 715 "%s: channel count more than 1, must be power of 2\n", __func__); 716 return; 717 } 718 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 719 ilog2(chan_cnt)); 720 req.chan_mask &= relaxed_mask; 721 } 722 723 req.channel = chan; 724 req.intf = pfvf->nix_rx_intf; 725 req.entry = index; 726 req.op = action.op; 727 req.hdr.pcifunc = 0; /* AF is requester */ 728 req.vf = pcifunc; 729 req.index = action.index; 730 req.match_id = action.match_id; 731 req.flow_key_alg = action.flow_key_alg; 732 733 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 734 } 735 736 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 737 int nixlf, bool enable) 738 { 739 struct npc_mcam *mcam = &rvu->hw->mcam; 740 int blkaddr, index; 741 742 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 743 if (blkaddr < 0) 744 return; 745 746 /* Get 'pcifunc' of PF device */ 747 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 748 749 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 750 nixlf, NIXLF_PROMISC_ENTRY); 751 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 752 } 753 754 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 755 int nixlf, u64 chan) 756 { 757 struct rvu_pfvf *pfvf; 758 struct npc_install_flow_req req = { 0 }; 759 struct npc_install_flow_rsp rsp = { 0 }; 760 struct npc_mcam *mcam = &rvu->hw->mcam; 761 struct rvu_hwinfo *hw = rvu->hw; 762 int blkaddr, index; 763 764 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 765 if (blkaddr < 0) 766 return; 767 768 /* Skip LBK VFs */ 769 if (is_afvf(pcifunc)) 770 return; 771 772 /* If pkt replication is not supported, 773 * then only PF is allowed to add a bcast match entry. 774 */ 775 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 776 return; 777 778 /* Get 'pcifunc' of PF device */ 779 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 780 pfvf = rvu_get_pfvf(rvu, pcifunc); 781 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 782 nixlf, NIXLF_BCAST_ENTRY); 783 784 if (!hw->cap.nix_rx_multicast) { 785 /* Early silicon doesn't support pkt replication, 786 * so install entry with UCAST action, so that PF 787 * receives all broadcast packets. 788 */ 789 req.op = NIX_RX_ACTIONOP_UCAST; 790 } else { 791 req.op = NIX_RX_ACTIONOP_MCAST; 792 req.index = pfvf->bcast_mce_idx; 793 } 794 795 eth_broadcast_addr((u8 *)&req.packet.dmac); 796 eth_broadcast_addr((u8 *)&req.mask.dmac); 797 req.features = BIT_ULL(NPC_DMAC); 798 req.channel = chan; 799 req.chan_mask = 0xFFFU; 800 req.intf = pfvf->nix_rx_intf; 801 req.entry = index; 802 req.hdr.pcifunc = 0; /* AF is requester */ 803 req.vf = pcifunc; 804 805 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 806 } 807 808 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 809 bool enable) 810 { 811 struct npc_mcam *mcam = &rvu->hw->mcam; 812 int blkaddr, index; 813 814 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 815 if (blkaddr < 0) 816 return; 817 818 /* Get 'pcifunc' of PF device */ 819 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 820 821 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 822 NIXLF_BCAST_ENTRY); 823 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 824 } 825 826 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 827 u64 chan) 828 { 829 struct npc_install_flow_req req = { 0 }; 830 struct npc_install_flow_rsp rsp = { 0 }; 831 struct npc_mcam *mcam = &rvu->hw->mcam; 832 struct rvu_hwinfo *hw = rvu->hw; 833 int blkaddr, ucast_idx, index; 834 u8 mac_addr[ETH_ALEN] = { 0 }; 835 struct nix_rx_action action; 836 struct rvu_pfvf *pfvf; 837 u16 vf_func; 838 839 /* Only CGX PF/VF can add allmulticast entry */ 840 if (is_afvf(pcifunc) && is_sdp_vf(pcifunc)) 841 return; 842 843 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 844 if (blkaddr < 0) 845 return; 846 847 /* Get 'pcifunc' of PF device */ 848 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 849 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 850 pfvf = rvu_get_pfvf(rvu, pcifunc); 851 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 852 nixlf, NIXLF_ALLMULTI_ENTRY); 853 854 /* If the corresponding PF's ucast action is RSS, 855 * use the same action for multicast entry also 856 */ 857 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 858 nixlf, NIXLF_UCAST_ENTRY); 859 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 860 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 861 blkaddr, ucast_idx); 862 863 if (action.op != NIX_RX_ACTIONOP_RSS) { 864 *(u64 *)&action = 0x00; 865 action.op = NIX_RX_ACTIONOP_UCAST; 866 action.pf_func = pcifunc; 867 } 868 869 /* RX_ACTION set to MCAST for CGX PF's */ 870 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 871 *(u64 *)&action = 0x00; 872 action.op = NIX_RX_ACTIONOP_MCAST; 873 action.index = pfvf->mcast_mce_idx; 874 } 875 876 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 877 ether_addr_copy(req.packet.dmac, mac_addr); 878 ether_addr_copy(req.mask.dmac, mac_addr); 879 req.features = BIT_ULL(NPC_DMAC); 880 881 /* For cn10k the upper two bits of the channel number are 882 * cpt channel number. with masking out these bits in the 883 * mcam entry, same entry used for NIX will allow packets 884 * received from cpt for parsing. 885 */ 886 if (!is_rvu_otx2(rvu)) 887 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 888 else 889 req.chan_mask = 0xFFFU; 890 891 req.channel = chan; 892 req.intf = pfvf->nix_rx_intf; 893 req.entry = index; 894 req.op = action.op; 895 req.hdr.pcifunc = 0; /* AF is requester */ 896 req.vf = pcifunc | vf_func; 897 req.index = action.index; 898 req.match_id = action.match_id; 899 req.flow_key_alg = action.flow_key_alg; 900 901 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 902 } 903 904 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 905 bool enable) 906 { 907 struct npc_mcam *mcam = &rvu->hw->mcam; 908 int blkaddr, index; 909 910 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 911 if (blkaddr < 0) 912 return; 913 914 /* Get 'pcifunc' of PF device */ 915 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 916 917 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 918 NIXLF_ALLMULTI_ENTRY); 919 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 920 } 921 922 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 923 int blkaddr, u16 pcifunc, u64 rx_action) 924 { 925 int actindex, index, bank, entry; 926 struct rvu_npc_mcam_rule *rule; 927 bool enable, update; 928 929 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 930 return; 931 932 mutex_lock(&mcam->lock); 933 for (index = 0; index < mcam->bmap_entries; index++) { 934 if (mcam->entry2target_pffunc[index] == pcifunc) { 935 update = true; 936 /* update not needed for the rules added via ntuple filters */ 937 list_for_each_entry(rule, &mcam->mcam_rules, list) { 938 if (rule->entry == index) 939 update = false; 940 } 941 if (!update) 942 continue; 943 bank = npc_get_bank(mcam, index); 944 actindex = index; 945 entry = index & (mcam->banksize - 1); 946 947 /* read vf flow entry enable status */ 948 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 949 actindex); 950 /* disable before mcam entry update */ 951 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 952 false); 953 /* update 'action' */ 954 rvu_write64(rvu, blkaddr, 955 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), 956 rx_action); 957 if (enable) 958 npc_enable_mcam_entry(rvu, mcam, blkaddr, 959 actindex, true); 960 } 961 } 962 mutex_unlock(&mcam->lock); 963 } 964 965 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 966 int group, int alg_idx, int mcam_index) 967 { 968 struct npc_mcam *mcam = &rvu->hw->mcam; 969 struct rvu_hwinfo *hw = rvu->hw; 970 struct nix_rx_action action; 971 int blkaddr, index, bank; 972 struct rvu_pfvf *pfvf; 973 974 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 975 if (blkaddr < 0) 976 return; 977 978 /* Check if this is for reserved default entry */ 979 if (mcam_index < 0) { 980 if (group != DEFAULT_RSS_CONTEXT_GROUP) 981 return; 982 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 983 nixlf, NIXLF_UCAST_ENTRY); 984 } else { 985 /* TODO: validate this mcam index */ 986 index = mcam_index; 987 } 988 989 if (index >= mcam->total_entries) 990 return; 991 992 bank = npc_get_bank(mcam, index); 993 index &= (mcam->banksize - 1); 994 995 *(u64 *)&action = rvu_read64(rvu, blkaddr, 996 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 997 /* Ignore if no action was set earlier */ 998 if (!*(u64 *)&action) 999 return; 1000 1001 action.op = NIX_RX_ACTIONOP_RSS; 1002 action.pf_func = pcifunc; 1003 action.index = group; 1004 action.flow_key_alg = alg_idx; 1005 1006 rvu_write64(rvu, blkaddr, 1007 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 1008 1009 /* update the VF flow rule action with the VF default entry action */ 1010 if (mcam_index < 0) 1011 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1012 *(u64 *)&action); 1013 1014 /* update the action change in default rule */ 1015 pfvf = rvu_get_pfvf(rvu, pcifunc); 1016 if (pfvf->def_ucast_rule) 1017 pfvf->def_ucast_rule->rx_action = action; 1018 1019 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1020 nixlf, NIXLF_PROMISC_ENTRY); 1021 1022 /* If PF's promiscuous entry is enabled, 1023 * Set RSS action for that entry as well 1024 */ 1025 if ((!hw->cap.nix_rx_multicast || !pfvf->use_mce_list) && 1026 is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 1027 bank = npc_get_bank(mcam, index); 1028 index &= (mcam->banksize - 1); 1029 1030 rvu_write64(rvu, blkaddr, 1031 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 1032 *(u64 *)&action); 1033 } 1034 } 1035 1036 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1037 int nixlf, int type, bool enable) 1038 { 1039 struct npc_mcam *mcam = &rvu->hw->mcam; 1040 struct rvu_hwinfo *hw = rvu->hw; 1041 struct nix_mce_list *mce_list; 1042 int index, blkaddr, mce_idx; 1043 struct rvu_pfvf *pfvf; 1044 1045 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1046 if (blkaddr < 0) 1047 return; 1048 1049 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1050 nixlf, type); 1051 1052 /* disable MCAM entry when packet replication is not supported by hw */ 1053 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1054 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1055 return; 1056 } 1057 1058 /* return incase mce list is not enabled */ 1059 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1060 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1061 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1062 return; 1063 1064 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1065 1066 nix_update_mce_list(rvu, pcifunc, mce_list, 1067 mce_idx, index, enable); 1068 if (enable) 1069 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1070 } 1071 1072 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1073 int nixlf, bool enable) 1074 { 1075 struct npc_mcam *mcam = &rvu->hw->mcam; 1076 int index, blkaddr; 1077 1078 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1079 if (blkaddr < 0) 1080 return; 1081 1082 /* Ucast MCAM match entry of this PF/VF */ 1083 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1084 nixlf, NIXLF_UCAST_ENTRY); 1085 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1086 1087 /* Nothing to do for VFs, on platforms where pkt replication 1088 * is not supported 1089 */ 1090 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1091 return; 1092 1093 /* add/delete pf_func to broadcast MCE list */ 1094 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1095 NIXLF_BCAST_ENTRY, enable); 1096 } 1097 1098 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1099 { 1100 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1101 1102 /* Delete multicast and promisc MCAM entries */ 1103 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1104 NIXLF_ALLMULTI_ENTRY, false); 1105 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1106 NIXLF_PROMISC_ENTRY, false); 1107 } 1108 1109 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1110 { 1111 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1112 * in set_rx_mode mbox handler. 1113 */ 1114 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1115 } 1116 1117 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1118 { 1119 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1120 struct npc_mcam *mcam = &rvu->hw->mcam; 1121 struct rvu_npc_mcam_rule *rule, *tmp; 1122 int blkaddr; 1123 1124 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1125 if (blkaddr < 0) 1126 return; 1127 1128 mutex_lock(&mcam->lock); 1129 1130 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1131 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1132 if (is_npc_intf_rx(rule->intf) && 1133 rule->rx_action.pf_func == pcifunc && 1134 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1135 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1136 rule->entry, false); 1137 rule->enable = false; 1138 /* Indicate that default rule is disabled */ 1139 if (rule->default_rule) { 1140 pfvf->def_ucast_rule = NULL; 1141 list_del(&rule->list); 1142 kfree(rule); 1143 } 1144 } 1145 } 1146 1147 mutex_unlock(&mcam->lock); 1148 1149 npc_mcam_disable_flows(rvu, pcifunc); 1150 1151 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1152 } 1153 1154 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1155 { 1156 struct npc_mcam *mcam = &rvu->hw->mcam; 1157 struct rvu_npc_mcam_rule *rule, *tmp; 1158 int blkaddr; 1159 1160 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1161 if (blkaddr < 0) 1162 return; 1163 1164 mutex_lock(&mcam->lock); 1165 1166 /* Free all MCAM entries owned by this 'pcifunc' */ 1167 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1168 1169 /* Free all MCAM counters owned by this 'pcifunc' */ 1170 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1171 1172 /* Delete MCAM entries owned by this 'pcifunc' */ 1173 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1174 if (rule->owner == pcifunc && !rule->default_rule) { 1175 list_del(&rule->list); 1176 kfree(rule); 1177 } 1178 } 1179 1180 mutex_unlock(&mcam->lock); 1181 1182 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1183 } 1184 1185 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1186 rvu_write64(rvu, blkaddr, \ 1187 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1188 1189 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1190 rvu_write64(rvu, blkaddr, \ 1191 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1192 1193 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1194 struct npc_mcam_kex *mkex, u8 intf) 1195 { 1196 int lid, lt, ld, fl; 1197 1198 if (is_npc_intf_tx(intf)) 1199 return; 1200 1201 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1202 mkex->keyx_cfg[NIX_INTF_RX]); 1203 1204 /* Program LDATA */ 1205 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1206 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1207 for (ld = 0; ld < NPC_MAX_LD; ld++) 1208 SET_KEX_LD(intf, lid, lt, ld, 1209 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1210 [lid][lt][ld]); 1211 } 1212 } 1213 /* Program LFLAGS */ 1214 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1215 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1216 SET_KEX_LDFLAGS(intf, ld, fl, 1217 mkex->intf_ld_flags[NIX_INTF_RX] 1218 [ld][fl]); 1219 } 1220 } 1221 1222 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1223 struct npc_mcam_kex *mkex, u8 intf) 1224 { 1225 int lid, lt, ld, fl; 1226 1227 if (is_npc_intf_rx(intf)) 1228 return; 1229 1230 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1231 mkex->keyx_cfg[NIX_INTF_TX]); 1232 1233 /* Program LDATA */ 1234 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1235 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1236 for (ld = 0; ld < NPC_MAX_LD; ld++) 1237 SET_KEX_LD(intf, lid, lt, ld, 1238 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1239 [lid][lt][ld]); 1240 } 1241 } 1242 /* Program LFLAGS */ 1243 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1244 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1245 SET_KEX_LDFLAGS(intf, ld, fl, 1246 mkex->intf_ld_flags[NIX_INTF_TX] 1247 [ld][fl]); 1248 } 1249 } 1250 1251 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1252 struct npc_mcam_kex *mkex) 1253 { 1254 struct rvu_hwinfo *hw = rvu->hw; 1255 u8 intf; 1256 int ld; 1257 1258 for (ld = 0; ld < NPC_MAX_LD; ld++) 1259 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1260 mkex->kex_ld_flags[ld]); 1261 1262 for (intf = 0; intf < hw->npc_intfs; intf++) { 1263 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1264 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1265 } 1266 } 1267 1268 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1269 u64 *size) 1270 { 1271 u64 prfl_addr, prfl_sz; 1272 1273 if (!rvu->fwdata) 1274 return -EINVAL; 1275 1276 prfl_addr = rvu->fwdata->mcam_addr; 1277 prfl_sz = rvu->fwdata->mcam_sz; 1278 1279 if (!prfl_addr || !prfl_sz) 1280 return -EINVAL; 1281 1282 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1283 if (!(*prfl_img_addr)) 1284 return -ENOMEM; 1285 1286 *size = prfl_sz; 1287 1288 return 0; 1289 } 1290 1291 /* strtoull of "mkexprof" with base:36 */ 1292 #define MKEX_END_SIGN 0xdeadbeef 1293 1294 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1295 const char *mkex_profile) 1296 { 1297 struct device *dev = &rvu->pdev->dev; 1298 struct npc_mcam_kex *mcam_kex; 1299 void __iomem *mkex_prfl_addr = NULL; 1300 u64 prfl_sz; 1301 int ret; 1302 1303 /* If user not selected mkex profile */ 1304 if (rvu->kpu_fwdata_sz || 1305 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1306 goto program_mkex; 1307 1308 /* Setting up the mapping for mkex profile image */ 1309 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1310 if (ret < 0) 1311 goto program_mkex; 1312 1313 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1314 1315 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1316 /* Compare with mkex mod_param name string */ 1317 if (mcam_kex->mkex_sign == MKEX_SIGN && 1318 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1319 /* Due to an errata (35786) in A0/B0 pass silicon, 1320 * parse nibble enable configuration has to be 1321 * identical for both Rx and Tx interfaces. 1322 */ 1323 if (!is_rvu_96xx_B0(rvu) || 1324 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1325 rvu->kpu.mkex = mcam_kex; 1326 goto program_mkex; 1327 } 1328 1329 mcam_kex++; 1330 prfl_sz -= sizeof(struct npc_mcam_kex); 1331 } 1332 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1333 1334 program_mkex: 1335 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1336 /* Program selected mkex profile */ 1337 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1338 if (mkex_prfl_addr) 1339 iounmap(mkex_prfl_addr); 1340 } 1341 1342 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1343 const struct npc_kpu_profile_action *kpuaction, 1344 int kpu, int entry, bool pkind) 1345 { 1346 struct npc_kpu_action0 action0 = {0}; 1347 struct npc_kpu_action1 action1 = {0}; 1348 u64 reg; 1349 1350 action1.errlev = kpuaction->errlev; 1351 action1.errcode = kpuaction->errcode; 1352 action1.dp0_offset = kpuaction->dp0_offset; 1353 action1.dp1_offset = kpuaction->dp1_offset; 1354 action1.dp2_offset = kpuaction->dp2_offset; 1355 1356 if (pkind) 1357 reg = NPC_AF_PKINDX_ACTION1(entry); 1358 else 1359 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1360 1361 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1362 1363 action0.byp_count = kpuaction->bypass_count; 1364 action0.capture_ena = kpuaction->cap_ena; 1365 action0.parse_done = kpuaction->parse_done; 1366 action0.next_state = kpuaction->next_state; 1367 action0.capture_lid = kpuaction->lid; 1368 action0.capture_ltype = kpuaction->ltype; 1369 action0.capture_flags = kpuaction->flags; 1370 action0.ptr_advance = kpuaction->ptr_advance; 1371 action0.var_len_offset = kpuaction->offset; 1372 action0.var_len_mask = kpuaction->mask; 1373 action0.var_len_right = kpuaction->right; 1374 action0.var_len_shift = kpuaction->shift; 1375 1376 if (pkind) 1377 reg = NPC_AF_PKINDX_ACTION0(entry); 1378 else 1379 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1380 1381 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1382 } 1383 1384 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1385 const struct npc_kpu_profile_cam *kpucam, 1386 int kpu, int entry) 1387 { 1388 struct npc_kpu_cam cam0 = {0}; 1389 struct npc_kpu_cam cam1 = {0}; 1390 1391 cam1.state = kpucam->state & kpucam->state_mask; 1392 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1393 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1394 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1395 1396 cam0.state = ~kpucam->state & kpucam->state_mask; 1397 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1398 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1399 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1400 1401 rvu_write64(rvu, blkaddr, 1402 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1403 rvu_write64(rvu, blkaddr, 1404 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1405 } 1406 1407 static inline u64 enable_mask(int count) 1408 { 1409 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1410 } 1411 1412 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1413 const struct npc_kpu_profile *profile) 1414 { 1415 int entry, num_entries, max_entries; 1416 u64 entry_mask; 1417 1418 if (profile->cam_entries != profile->action_entries) { 1419 dev_err(rvu->dev, 1420 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1421 kpu, profile->cam_entries, profile->action_entries); 1422 } 1423 1424 max_entries = rvu->hw->npc_kpu_entries; 1425 1426 /* Program CAM match entries for previous KPU extracted data */ 1427 num_entries = min_t(int, profile->cam_entries, max_entries); 1428 for (entry = 0; entry < num_entries; entry++) 1429 npc_config_kpucam(rvu, blkaddr, 1430 &profile->cam[entry], kpu, entry); 1431 1432 /* Program this KPU's actions */ 1433 num_entries = min_t(int, profile->action_entries, max_entries); 1434 for (entry = 0; entry < num_entries; entry++) 1435 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1436 kpu, entry, false); 1437 1438 /* Enable all programmed entries */ 1439 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1440 entry_mask = enable_mask(num_entries); 1441 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1442 if (!rvu->kpu.custom) 1443 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1444 rvu_write64(rvu, blkaddr, 1445 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1446 if (num_entries > 64) { 1447 rvu_write64(rvu, blkaddr, 1448 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1449 enable_mask(num_entries - 64)); 1450 } 1451 1452 /* Enable this KPU */ 1453 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1454 } 1455 1456 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1457 { 1458 profile->custom = 0; 1459 profile->name = def_pfl_name; 1460 profile->version = NPC_KPU_PROFILE_VER; 1461 profile->ikpu = ikpu_action_entries; 1462 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1463 profile->kpu = npc_kpu_profiles; 1464 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1465 profile->lt_def = &npc_lt_defaults; 1466 profile->mkex = &npc_mkex_default; 1467 1468 return 0; 1469 } 1470 1471 static int npc_apply_custom_kpu(struct rvu *rvu, 1472 struct npc_kpu_profile_adapter *profile) 1473 { 1474 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1475 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1476 struct npc_kpu_profile_action *action; 1477 struct npc_kpu_profile_cam *cam; 1478 struct npc_kpu_fwdata *fw_kpu; 1479 int entries; 1480 u16 kpu, entry; 1481 1482 if (rvu->kpu_fwdata_sz < hdr_sz) { 1483 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1484 return -EINVAL; 1485 } 1486 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1487 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1488 fw->signature); 1489 return -EINVAL; 1490 } 1491 /* Verify if the using known profile structure */ 1492 if (NPC_KPU_VER_MAJ(profile->version) > 1493 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1494 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1495 NPC_KPU_VER_MAJ(profile->version), 1496 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1497 return -EINVAL; 1498 } 1499 /* Verify if profile is aligned with the required kernel changes */ 1500 if (NPC_KPU_VER_MIN(profile->version) < 1501 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1502 dev_warn(rvu->dev, 1503 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1504 NPC_KPU_VER_MAJ(profile->version), 1505 NPC_KPU_VER_MIN(profile->version), 1506 NPC_KPU_VER_PATCH(profile->version), 1507 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1508 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1509 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1510 return -EINVAL; 1511 } 1512 /* Verify if profile fits the HW */ 1513 if (fw->kpus > profile->kpus) { 1514 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1515 profile->kpus); 1516 return -EINVAL; 1517 } 1518 1519 profile->custom = 1; 1520 profile->name = fw->name; 1521 profile->version = le64_to_cpu(fw->version); 1522 profile->mkex = &fw->mkex; 1523 profile->lt_def = &fw->lt_def; 1524 1525 for (kpu = 0; kpu < fw->kpus; kpu++) { 1526 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1527 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1528 dev_warn(rvu->dev, 1529 "Too many custom entries on KPU%d: %d > %d\n", 1530 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1531 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1532 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1533 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1534 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1535 offset += fw_kpu->entries * sizeof(*action); 1536 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1537 dev_warn(rvu->dev, 1538 "Profile size mismatch on KPU%i parsing.\n", 1539 kpu + 1); 1540 return -EINVAL; 1541 } 1542 for (entry = 0; entry < entries; entry++) { 1543 profile->kpu[kpu].cam[entry] = cam[entry]; 1544 profile->kpu[kpu].action[entry] = action[entry]; 1545 } 1546 } 1547 1548 return 0; 1549 } 1550 1551 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1552 u64 prfl_sz, const char *kpu_profile) 1553 { 1554 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1555 int rc = -EINVAL; 1556 1557 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1558 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1559 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1560 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1561 kpu_profile); 1562 rvu->kpu_fwdata = kpu_data; 1563 rvu->kpu_fwdata_sz = prfl_sz; 1564 rvu->kpu_prfl_addr = prfl_addr; 1565 rc = 0; 1566 } 1567 1568 return rc; 1569 } 1570 1571 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1572 const char *kpu_profile) 1573 { 1574 struct npc_coalesced_kpu_prfl *img_data = NULL; 1575 int i = 0, rc = -EINVAL; 1576 void __iomem *kpu_prfl_addr; 1577 u16 offset; 1578 1579 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1580 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1581 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1582 /* Loaded profile is a single KPU profile. */ 1583 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1584 prfl_sz, kpu_profile); 1585 goto done; 1586 } 1587 1588 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1589 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1590 (img_data->num_prfl * sizeof(uint16_t)); 1591 /* Check if mapped image is coalesced image. */ 1592 while (i < img_data->num_prfl) { 1593 /* Profile image offsets are rounded up to next 8 multiple.*/ 1594 offset = ALIGN_8B_CEIL(offset); 1595 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1596 offset); 1597 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1598 img_data->prfl_sz[i], kpu_profile); 1599 if (!rc) 1600 break; 1601 /* Calculating offset of profile image based on profile size.*/ 1602 offset += img_data->prfl_sz[i]; 1603 i++; 1604 } 1605 done: 1606 return rc; 1607 } 1608 1609 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1610 { 1611 int ret = -EINVAL; 1612 u64 prfl_sz; 1613 1614 /* Setting up the mapping for NPC profile image */ 1615 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1616 if (ret < 0) 1617 goto done; 1618 1619 /* Detect if profile is coalesced or single KPU profile and load */ 1620 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1621 if (ret == 0) 1622 goto done; 1623 1624 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1625 if (rvu->kpu_prfl_addr) { 1626 iounmap(rvu->kpu_prfl_addr); 1627 rvu->kpu_prfl_addr = NULL; 1628 rvu->kpu_fwdata_sz = 0; 1629 rvu->kpu_fwdata = NULL; 1630 } 1631 1632 done: 1633 return ret; 1634 } 1635 1636 static void npc_load_kpu_profile(struct rvu *rvu) 1637 { 1638 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1639 const char *kpu_profile = rvu->kpu_pfl_name; 1640 const struct firmware *fw = NULL; 1641 bool retry_fwdb = false; 1642 1643 /* If user not specified profile customization */ 1644 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1645 goto revert_to_default; 1646 /* First prepare default KPU, then we'll customize top entries. */ 1647 npc_prepare_default_kpu(profile); 1648 1649 /* Order of preceedence for load loading NPC profile (high to low) 1650 * Firmware binary in filesystem. 1651 * Firmware database method. 1652 * Default KPU profile. 1653 */ 1654 if (!request_firmware(&fw, kpu_profile, rvu->dev)) { 1655 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1656 kpu_profile); 1657 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1658 if (rvu->kpu_fwdata) { 1659 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1660 rvu->kpu_fwdata_sz = fw->size; 1661 } 1662 release_firmware(fw); 1663 retry_fwdb = true; 1664 goto program_kpu; 1665 } 1666 1667 load_image_fwdb: 1668 /* Loading the KPU profile using firmware database */ 1669 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1670 goto revert_to_default; 1671 1672 program_kpu: 1673 /* Apply profile customization if firmware was loaded. */ 1674 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1675 /* If image from firmware filesystem fails to load or invalid 1676 * retry with firmware database method. 1677 */ 1678 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1679 /* Loading image from firmware database failed. */ 1680 if (rvu->kpu_prfl_addr) { 1681 iounmap(rvu->kpu_prfl_addr); 1682 rvu->kpu_prfl_addr = NULL; 1683 } else { 1684 kfree(rvu->kpu_fwdata); 1685 } 1686 rvu->kpu_fwdata = NULL; 1687 rvu->kpu_fwdata_sz = 0; 1688 if (retry_fwdb) { 1689 retry_fwdb = false; 1690 goto load_image_fwdb; 1691 } 1692 } 1693 1694 dev_warn(rvu->dev, 1695 "Can't load KPU profile %s. Using default.\n", 1696 kpu_profile); 1697 kfree(rvu->kpu_fwdata); 1698 rvu->kpu_fwdata = NULL; 1699 goto revert_to_default; 1700 } 1701 1702 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1703 profile->name, NPC_KPU_VER_MAJ(profile->version), 1704 NPC_KPU_VER_MIN(profile->version), 1705 NPC_KPU_VER_PATCH(profile->version)); 1706 1707 return; 1708 1709 revert_to_default: 1710 npc_prepare_default_kpu(profile); 1711 } 1712 1713 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1714 { 1715 struct rvu_hwinfo *hw = rvu->hw; 1716 int num_pkinds, num_kpus, idx; 1717 1718 /* Disable all KPUs and their entries */ 1719 for (idx = 0; idx < hw->npc_kpus; idx++) { 1720 rvu_write64(rvu, blkaddr, 1721 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1722 rvu_write64(rvu, blkaddr, 1723 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1724 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1725 } 1726 1727 /* Load and customize KPU profile. */ 1728 npc_load_kpu_profile(rvu); 1729 1730 /* First program IKPU profile i.e PKIND configs. 1731 * Check HW max count to avoid configuring junk or 1732 * writing to unsupported CSR addresses. 1733 */ 1734 num_pkinds = rvu->kpu.pkinds; 1735 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); 1736 1737 for (idx = 0; idx < num_pkinds; idx++) 1738 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1739 1740 /* Program KPU CAM and Action profiles */ 1741 num_kpus = rvu->kpu.kpus; 1742 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1743 1744 for (idx = 0; idx < num_kpus; idx++) 1745 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1746 } 1747 1748 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1749 { 1750 int nixlf_count = rvu_get_nixlf_count(rvu); 1751 struct npc_mcam *mcam = &rvu->hw->mcam; 1752 int rsvd, err; 1753 u16 index; 1754 int cntr; 1755 u64 cfg; 1756 1757 /* Actual number of MCAM entries vary by entry size */ 1758 cfg = (rvu_read64(rvu, blkaddr, 1759 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1760 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1761 mcam->keysize = cfg; 1762 1763 /* Number of banks combined per MCAM entry */ 1764 if (cfg == NPC_MCAM_KEY_X4) 1765 mcam->banks_per_entry = 4; 1766 else if (cfg == NPC_MCAM_KEY_X2) 1767 mcam->banks_per_entry = 2; 1768 else 1769 mcam->banks_per_entry = 1; 1770 1771 /* Reserve one MCAM entry for each of the NIX LF to 1772 * guarantee space to install default matching DMAC rule. 1773 * Also reserve 2 MCAM entries for each PF for default 1774 * channel based matching or 'bcast & promisc' matching to 1775 * support BCAST and PROMISC modes of operation for PFs. 1776 * PF0 is excluded. 1777 */ 1778 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1779 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1780 if (mcam->total_entries <= rsvd) { 1781 dev_warn(rvu->dev, 1782 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1783 mcam->total_entries); 1784 return -ENOMEM; 1785 } 1786 1787 mcam->bmap_entries = mcam->total_entries - rsvd; 1788 mcam->nixlf_offset = mcam->bmap_entries; 1789 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1790 1791 /* Allocate bitmaps for managing MCAM entries */ 1792 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1793 sizeof(long), GFP_KERNEL); 1794 if (!mcam->bmap) 1795 return -ENOMEM; 1796 1797 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1798 BITS_TO_LONGS(mcam->bmap_entries), 1799 sizeof(long), GFP_KERNEL); 1800 if (!mcam->bmap_reverse) 1801 return -ENOMEM; 1802 1803 mcam->bmap_fcnt = mcam->bmap_entries; 1804 1805 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1806 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1807 sizeof(u16), GFP_KERNEL); 1808 if (!mcam->entry2pfvf_map) 1809 return -ENOMEM; 1810 1811 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1812 * allocations and another 1/8th at the top for high priority 1813 * allocations. 1814 */ 1815 mcam->lprio_count = mcam->bmap_entries / 8; 1816 if (mcam->lprio_count > BITS_PER_LONG) 1817 mcam->lprio_count = round_down(mcam->lprio_count, 1818 BITS_PER_LONG); 1819 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1820 mcam->hprio_count = mcam->lprio_count; 1821 mcam->hprio_end = mcam->hprio_count; 1822 1823 1824 /* Allocate bitmap for managing MCAM counters and memory 1825 * for saving counter to RVU PFFUNC allocation mapping. 1826 */ 1827 err = rvu_alloc_bitmap(&mcam->counters); 1828 if (err) 1829 return err; 1830 1831 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1832 sizeof(u16), GFP_KERNEL); 1833 if (!mcam->cntr2pfvf_map) 1834 goto free_mem; 1835 1836 /* Alloc memory for MCAM entry to counter mapping and for tracking 1837 * counter's reference count. 1838 */ 1839 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1840 sizeof(u16), GFP_KERNEL); 1841 if (!mcam->entry2cntr_map) 1842 goto free_mem; 1843 1844 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1845 sizeof(u16), GFP_KERNEL); 1846 if (!mcam->cntr_refcnt) 1847 goto free_mem; 1848 1849 /* Alloc memory for saving target device of mcam rule */ 1850 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1851 sizeof(u16), GFP_KERNEL); 1852 if (!mcam->entry2target_pffunc) 1853 goto free_mem; 1854 1855 for (index = 0; index < mcam->bmap_entries; index++) { 1856 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1857 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1858 } 1859 1860 for (cntr = 0; cntr < mcam->counters.max; cntr++) 1861 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1862 1863 mutex_init(&mcam->lock); 1864 1865 return 0; 1866 1867 free_mem: 1868 kfree(mcam->counters.bmap); 1869 return -ENOMEM; 1870 } 1871 1872 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1873 { 1874 struct npc_pkind *pkind = &rvu->hw->pkind; 1875 struct npc_mcam *mcam = &rvu->hw->mcam; 1876 struct rvu_hwinfo *hw = rvu->hw; 1877 u64 npc_const, npc_const1; 1878 u64 npc_const2 = 0; 1879 1880 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1881 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1882 if (npc_const1 & BIT_ULL(63)) 1883 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1884 1885 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT; 1886 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; 1887 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1888 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1889 hw->npc_intfs = npc_const & 0xFULL; 1890 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1891 1892 mcam->banks = (npc_const >> 44) & 0xFULL; 1893 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1894 hw->npc_stat_ena = BIT_ULL(9); 1895 /* Extended set */ 1896 if (npc_const2) { 1897 hw->npc_ext_set = true; 1898 /* 96xx supports only match_stats and npc_counters 1899 * reflected in NPC_AF_CONST reg. 1900 * STAT_SEL and ENA are at [0:8] and 9 bit positions. 1901 * 98xx has both match_stat and ext and npc_counter 1902 * reflected in NPC_AF_CONST2 1903 * STAT_SEL_EXT added at [12:14] bit position. 1904 * cn10k supports only ext and hence npc_counters in 1905 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2. 1906 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 63 1907 */ 1908 if (!hw->npc_counters) 1909 hw->npc_stat_ena = BIT_ULL(63); 1910 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1911 mcam->banksize = npc_const2 & 0xFFFFULL; 1912 } 1913 1914 mcam->counters.max = hw->npc_counters; 1915 } 1916 1917 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1918 { 1919 struct npc_mcam *mcam = &rvu->hw->mcam; 1920 struct rvu_hwinfo *hw = rvu->hw; 1921 u64 nibble_ena, rx_kex, tx_kex; 1922 u8 intf; 1923 1924 /* Reserve last counter for MCAM RX miss action which is set to 1925 * drop packet. This way we will know how many pkts didn't match 1926 * any MCAM entry. 1927 */ 1928 mcam->counters.max--; 1929 mcam->rx_miss_act_cntr = mcam->counters.max; 1930 1931 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1932 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1933 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1934 1935 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1936 if (nibble_ena) { 1937 tx_kex &= ~NPC_PARSE_NIBBLE; 1938 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1939 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1940 } 1941 1942 /* Configure RX interfaces */ 1943 for (intf = 0; intf < hw->npc_intfs; intf++) { 1944 if (is_npc_intf_tx(intf)) 1945 continue; 1946 1947 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1948 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1949 rx_kex); 1950 1951 /* If MCAM lookup doesn't result in a match, drop the received 1952 * packet. And map this action to a counter to count dropped 1953 * packets. 1954 */ 1955 rvu_write64(rvu, blkaddr, 1956 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1957 1958 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1959 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1960 */ 1961 rvu_write64(rvu, blkaddr, 1962 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1963 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1964 hw->npc_stat_ena | mcam->rx_miss_act_cntr); 1965 } 1966 1967 /* Configure TX interfaces */ 1968 for (intf = 0; intf < hw->npc_intfs; intf++) { 1969 if (is_npc_intf_rx(intf)) 1970 continue; 1971 1972 /* Extract Ltypes LID_LA to LID_LE */ 1973 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1974 tx_kex); 1975 1976 /* Set TX miss action to UCAST_DEFAULT i.e 1977 * transmit the packet on NIX LF SQ's default channel. 1978 */ 1979 rvu_write64(rvu, blkaddr, 1980 NPC_AF_INTFX_MISS_ACT(intf), 1981 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1982 } 1983 } 1984 1985 int rvu_npc_init(struct rvu *rvu) 1986 { 1987 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1988 struct npc_pkind *pkind = &rvu->hw->pkind; 1989 struct npc_mcam *mcam = &rvu->hw->mcam; 1990 int blkaddr, entry, bank, err; 1991 1992 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1993 if (blkaddr < 0) { 1994 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1995 return -ENODEV; 1996 } 1997 1998 rvu_npc_hw_init(rvu, blkaddr); 1999 2000 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 2001 for (bank = 0; bank < mcam->banks; bank++) { 2002 for (entry = 0; entry < mcam->banksize; entry++) 2003 rvu_write64(rvu, blkaddr, 2004 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 2005 } 2006 2007 err = rvu_alloc_bitmap(&pkind->rsrc); 2008 if (err) 2009 return err; 2010 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0', 2011 * no need to configure PKIND for all LBKs separately. 2012 */ 2013 rvu_alloc_rsrc(&pkind->rsrc); 2014 2015 /* Allocate mem for pkind to PF and channel mapping info */ 2016 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 2017 sizeof(u32), GFP_KERNEL); 2018 if (!pkind->pfchan_map) 2019 return -ENOMEM; 2020 2021 /* Configure KPU profile */ 2022 npc_parser_profile_init(rvu, blkaddr); 2023 2024 /* Config Outer L2, IPv4's NPC layer info */ 2025 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 2026 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 2027 kpu->lt_def->pck_ol2.ltype_mask); 2028 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 2029 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 2030 kpu->lt_def->pck_oip4.ltype_mask); 2031 2032 /* Config Inner IPV4 NPC layer info */ 2033 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2034 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2035 kpu->lt_def->pck_iip4.ltype_mask); 2036 2037 /* Enable below for Rx pkts. 2038 * - Outer IPv4 header checksum validation. 2039 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B]. 2040 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M]. 2041 * - Inner IPv4 header checksum validation. 2042 * - Set non zero checksum error code value 2043 */ 2044 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2045 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2046 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) | 2047 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1)); 2048 2049 rvu_npc_setup_interfaces(rvu, blkaddr); 2050 2051 /* Configure MKEX profile */ 2052 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2053 2054 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2055 if (err) 2056 return err; 2057 2058 err = npc_flow_steering_init(rvu, blkaddr); 2059 if (err) { 2060 dev_err(rvu->dev, 2061 "Incorrect mkex profile loaded using default mkex\n"); 2062 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2063 } 2064 2065 return 0; 2066 } 2067 2068 void rvu_npc_freemem(struct rvu *rvu) 2069 { 2070 struct npc_pkind *pkind = &rvu->hw->pkind; 2071 struct npc_mcam *mcam = &rvu->hw->mcam; 2072 2073 kfree(pkind->rsrc.bmap); 2074 kfree(mcam->counters.bmap); 2075 if (rvu->kpu_prfl_addr) 2076 iounmap(rvu->kpu_prfl_addr); 2077 else 2078 kfree(rvu->kpu_fwdata); 2079 mutex_destroy(&mcam->lock); 2080 } 2081 2082 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2083 int blkaddr, int *alloc_cnt, 2084 int *enable_cnt) 2085 { 2086 struct npc_mcam *mcam = &rvu->hw->mcam; 2087 int entry; 2088 2089 *alloc_cnt = 0; 2090 *enable_cnt = 0; 2091 2092 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2093 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2094 (*alloc_cnt)++; 2095 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2096 (*enable_cnt)++; 2097 } 2098 } 2099 } 2100 2101 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2102 int blkaddr, int *alloc_cnt, 2103 int *enable_cnt) 2104 { 2105 struct npc_mcam *mcam = &rvu->hw->mcam; 2106 int cntr; 2107 2108 *alloc_cnt = 0; 2109 *enable_cnt = 0; 2110 2111 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2112 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2113 (*alloc_cnt)++; 2114 if (mcam->cntr_refcnt[cntr]) 2115 (*enable_cnt)++; 2116 } 2117 } 2118 } 2119 2120 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 2121 u16 pcifunc, int entry) 2122 { 2123 /* verify AF installed entries */ 2124 if (is_pffunc_af(pcifunc)) 2125 return 0; 2126 /* Verify if entry is valid and if it is indeed 2127 * allocated to the requesting PFFUNC. 2128 */ 2129 if (entry >= mcam->bmap_entries) 2130 return NPC_MCAM_INVALID_REQ; 2131 2132 if (pcifunc != mcam->entry2pfvf_map[entry]) 2133 return NPC_MCAM_PERM_DENIED; 2134 2135 return 0; 2136 } 2137 2138 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2139 u16 pcifunc, int cntr) 2140 { 2141 /* Verify if counter is valid and if it is indeed 2142 * allocated to the requesting PFFUNC. 2143 */ 2144 if (cntr >= mcam->counters.max) 2145 return NPC_MCAM_INVALID_REQ; 2146 2147 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2148 return NPC_MCAM_PERM_DENIED; 2149 2150 return 0; 2151 } 2152 2153 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2154 int blkaddr, u16 entry, u16 cntr) 2155 { 2156 u16 index = entry & (mcam->banksize - 1); 2157 u32 bank = npc_get_bank(mcam, entry); 2158 struct rvu_hwinfo *hw = rvu->hw; 2159 2160 /* Set mapping and increment counter's refcnt */ 2161 mcam->entry2cntr_map[entry] = cntr; 2162 mcam->cntr_refcnt[cntr]++; 2163 /* Enable stats */ 2164 rvu_write64(rvu, blkaddr, 2165 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2166 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr); 2167 } 2168 2169 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2170 struct npc_mcam *mcam, 2171 int blkaddr, u16 entry, u16 cntr) 2172 { 2173 u16 index = entry & (mcam->banksize - 1); 2174 u32 bank = npc_get_bank(mcam, entry); 2175 2176 /* Remove mapping and reduce counter's refcnt */ 2177 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2178 mcam->cntr_refcnt[cntr]--; 2179 /* Disable stats */ 2180 rvu_write64(rvu, blkaddr, 2181 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2182 } 2183 2184 /* Sets MCAM entry in bitmap as used. Update 2185 * reverse bitmap too. Should be called with 2186 * 'mcam->lock' held. 2187 */ 2188 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2189 { 2190 u16 entry, rentry; 2191 2192 entry = index; 2193 rentry = mcam->bmap_entries - index - 1; 2194 2195 __set_bit(entry, mcam->bmap); 2196 __set_bit(rentry, mcam->bmap_reverse); 2197 mcam->bmap_fcnt--; 2198 } 2199 2200 /* Sets MCAM entry in bitmap as free. Update 2201 * reverse bitmap too. Should be called with 2202 * 'mcam->lock' held. 2203 */ 2204 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2205 { 2206 u16 entry, rentry; 2207 2208 entry = index; 2209 rentry = mcam->bmap_entries - index - 1; 2210 2211 __clear_bit(entry, mcam->bmap); 2212 __clear_bit(rentry, mcam->bmap_reverse); 2213 mcam->bmap_fcnt++; 2214 } 2215 2216 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2217 int blkaddr, u16 pcifunc) 2218 { 2219 u16 index, cntr; 2220 2221 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2222 for (index = 0; index < mcam->bmap_entries; index++) { 2223 if (mcam->entry2pfvf_map[index] == pcifunc) { 2224 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2225 /* Free the entry in bitmap */ 2226 npc_mcam_clear_bit(mcam, index); 2227 /* Disable the entry */ 2228 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2229 2230 /* Update entry2counter mapping */ 2231 cntr = mcam->entry2cntr_map[index]; 2232 if (cntr != NPC_MCAM_INVALID_MAP) 2233 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2234 blkaddr, index, 2235 cntr); 2236 mcam->entry2target_pffunc[index] = 0x0; 2237 } 2238 } 2239 } 2240 2241 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2242 u16 pcifunc) 2243 { 2244 u16 cntr; 2245 2246 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2247 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2248 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2249 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2250 mcam->cntr_refcnt[cntr] = 0; 2251 rvu_free_rsrc(&mcam->counters, cntr); 2252 /* This API is expected to be called after freeing 2253 * MCAM entries, which inturn will remove 2254 * 'entry to counter' mapping. 2255 * No need to do it again. 2256 */ 2257 } 2258 } 2259 } 2260 2261 /* Find area of contiguous free entries of size 'nr'. 2262 * If not found return max contiguous free entries available. 2263 */ 2264 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2265 u16 nr, u16 *max_area) 2266 { 2267 u16 max_area_start = 0; 2268 u16 index, next, end; 2269 2270 *max_area = 0; 2271 2272 again: 2273 index = find_next_zero_bit(map, size, start); 2274 if (index >= size) 2275 return max_area_start; 2276 2277 end = ((index + nr) >= size) ? size : index + nr; 2278 next = find_next_bit(map, end, index); 2279 if (*max_area < (next - index)) { 2280 *max_area = next - index; 2281 max_area_start = index; 2282 } 2283 2284 if (next < end) { 2285 start = next + 1; 2286 goto again; 2287 } 2288 2289 return max_area_start; 2290 } 2291 2292 /* Find number of free MCAM entries available 2293 * within range i.e in between 'start' and 'end'. 2294 */ 2295 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2296 { 2297 u16 index, next; 2298 u16 fcnt = 0; 2299 2300 again: 2301 if (start >= end) 2302 return fcnt; 2303 2304 index = find_next_zero_bit(map, end, start); 2305 if (index >= end) 2306 return fcnt; 2307 2308 next = find_next_bit(map, end, index); 2309 if (next <= end) { 2310 fcnt += next - index; 2311 start = next + 1; 2312 goto again; 2313 } 2314 2315 fcnt += end - index; 2316 return fcnt; 2317 } 2318 2319 static void 2320 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2321 struct npc_mcam_alloc_entry_req *req, 2322 u16 *start, u16 *end, bool *reverse) 2323 { 2324 u16 fcnt; 2325 2326 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2327 goto hprio; 2328 2329 /* For a low priority entry allocation 2330 * - If reference entry is not in hprio zone then 2331 * search range: ref_entry to end. 2332 * - If reference entry is in hprio zone and if 2333 * request can be accomodated in non-hprio zone then 2334 * search range: 'start of middle zone' to 'end' 2335 * - else search in reverse, so that less number of hprio 2336 * zone entries are allocated. 2337 */ 2338 2339 *reverse = false; 2340 *start = req->ref_entry + 1; 2341 *end = mcam->bmap_entries; 2342 2343 if (req->ref_entry >= mcam->hprio_end) 2344 return; 2345 2346 fcnt = npc_mcam_get_free_count(mcam->bmap, 2347 mcam->hprio_end, mcam->bmap_entries); 2348 if (fcnt > req->count) 2349 *start = mcam->hprio_end; 2350 else 2351 *reverse = true; 2352 return; 2353 2354 hprio: 2355 /* For a high priority entry allocation, search is always 2356 * in reverse to preserve hprio zone entries. 2357 * - If reference entry is not in lprio zone then 2358 * search range: 0 to ref_entry. 2359 * - If reference entry is in lprio zone and if 2360 * request can be accomodated in middle zone then 2361 * search range: 'hprio_end' to 'lprio_start' 2362 */ 2363 2364 *reverse = true; 2365 *start = 0; 2366 *end = req->ref_entry; 2367 2368 if (req->ref_entry <= mcam->lprio_start) 2369 return; 2370 2371 fcnt = npc_mcam_get_free_count(mcam->bmap, 2372 mcam->hprio_end, mcam->lprio_start); 2373 if (fcnt < req->count) 2374 return; 2375 *start = mcam->hprio_end; 2376 *end = mcam->lprio_start; 2377 } 2378 2379 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2380 struct npc_mcam_alloc_entry_req *req, 2381 struct npc_mcam_alloc_entry_rsp *rsp) 2382 { 2383 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2384 u16 fcnt, hp_fcnt, lp_fcnt; 2385 u16 start, end, index; 2386 int entry, next_start; 2387 bool reverse = false; 2388 unsigned long *bmap; 2389 u16 max_contig; 2390 2391 mutex_lock(&mcam->lock); 2392 2393 /* Check if there are any free entries */ 2394 if (!mcam->bmap_fcnt) { 2395 mutex_unlock(&mcam->lock); 2396 return NPC_MCAM_ALLOC_FAILED; 2397 } 2398 2399 /* MCAM entries are divided into high priority, middle and 2400 * low priority zones. Idea is to not allocate top and lower 2401 * most entries as much as possible, this is to increase 2402 * probability of honouring priority allocation requests. 2403 * 2404 * Two bitmaps are used for mcam entry management, 2405 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2406 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2407 * 2408 * Reverse bitmap is used to allocate entries 2409 * - when a higher priority entry is requested 2410 * - when available free entries are less. 2411 * Lower priority ones out of avaialble free entries are always 2412 * chosen when 'high vs low' question arises. 2413 */ 2414 2415 /* Get the search range for priority allocation request */ 2416 if (req->priority) { 2417 npc_get_mcam_search_range_priority(mcam, req, 2418 &start, &end, &reverse); 2419 goto alloc; 2420 } 2421 2422 /* For a VF base MCAM match rule is set by its PF. And all the 2423 * further MCAM rules installed by VF on its own are 2424 * concatenated with the base rule set by its PF. Hence PF entries 2425 * should be at lower priority compared to VF entries. Otherwise 2426 * base rule is hit always and rules installed by VF will be of 2427 * no use. Hence if the request is from PF and NOT a priority 2428 * allocation request then allocate low priority entries. 2429 */ 2430 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 2431 goto lprio_alloc; 2432 2433 /* Find out the search range for non-priority allocation request 2434 * 2435 * Get MCAM free entry count in middle zone. 2436 */ 2437 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2438 mcam->lprio_start, 2439 mcam->bmap_entries); 2440 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2441 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2442 2443 /* Check if request can be accomodated in the middle zone */ 2444 if (fcnt > req->count) { 2445 start = mcam->hprio_end; 2446 end = mcam->lprio_start; 2447 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2448 /* Expand search zone from half of hprio zone to 2449 * half of lprio zone. 2450 */ 2451 start = mcam->hprio_end / 2; 2452 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2453 reverse = true; 2454 } else { 2455 /* Not enough free entries, search all entries in reverse, 2456 * so that low priority ones will get used up. 2457 */ 2458 lprio_alloc: 2459 reverse = true; 2460 start = 0; 2461 end = mcam->bmap_entries; 2462 } 2463 2464 alloc: 2465 if (reverse) { 2466 bmap = mcam->bmap_reverse; 2467 start = mcam->bmap_entries - start; 2468 end = mcam->bmap_entries - end; 2469 swap(start, end); 2470 } else { 2471 bmap = mcam->bmap; 2472 } 2473 2474 if (req->contig) { 2475 /* Allocate requested number of contiguous entries, if 2476 * unsuccessful find max contiguous entries available. 2477 */ 2478 index = npc_mcam_find_zero_area(bmap, end, start, 2479 req->count, &max_contig); 2480 rsp->count = max_contig; 2481 if (reverse) 2482 rsp->entry = mcam->bmap_entries - index - max_contig; 2483 else 2484 rsp->entry = index; 2485 } else { 2486 /* Allocate requested number of non-contiguous entries, 2487 * if unsuccessful allocate as many as possible. 2488 */ 2489 rsp->count = 0; 2490 next_start = start; 2491 for (entry = 0; entry < req->count; entry++) { 2492 index = find_next_zero_bit(bmap, end, next_start); 2493 if (index >= end) 2494 break; 2495 2496 next_start = start + (index - start) + 1; 2497 2498 /* Save the entry's index */ 2499 if (reverse) 2500 index = mcam->bmap_entries - index - 1; 2501 entry_list[entry] = index; 2502 rsp->count++; 2503 } 2504 } 2505 2506 /* If allocating requested no of entries is unsucessful, 2507 * expand the search range to full bitmap length and retry. 2508 */ 2509 if (!req->priority && (rsp->count < req->count) && 2510 ((end - start) != mcam->bmap_entries)) { 2511 reverse = true; 2512 start = 0; 2513 end = mcam->bmap_entries; 2514 goto alloc; 2515 } 2516 2517 /* For priority entry allocation requests, if allocation is 2518 * failed then expand search to max possible range and retry. 2519 */ 2520 if (req->priority && rsp->count < req->count) { 2521 if (req->priority == NPC_MCAM_LOWER_PRIO && 2522 (start != (req->ref_entry + 1))) { 2523 start = req->ref_entry + 1; 2524 end = mcam->bmap_entries; 2525 reverse = false; 2526 goto alloc; 2527 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2528 ((end - start) != req->ref_entry)) { 2529 start = 0; 2530 end = req->ref_entry; 2531 reverse = true; 2532 goto alloc; 2533 } 2534 } 2535 2536 /* Copy MCAM entry indices into mbox response entry_list. 2537 * Requester always expects indices in ascending order, so 2538 * so reverse the list if reverse bitmap is used for allocation. 2539 */ 2540 if (!req->contig && rsp->count) { 2541 index = 0; 2542 for (entry = rsp->count - 1; entry >= 0; entry--) { 2543 if (reverse) 2544 rsp->entry_list[index++] = entry_list[entry]; 2545 else 2546 rsp->entry_list[entry] = entry_list[entry]; 2547 } 2548 } 2549 2550 /* Mark the allocated entries as used and set nixlf mapping */ 2551 for (entry = 0; entry < rsp->count; entry++) { 2552 index = req->contig ? 2553 (rsp->entry + entry) : rsp->entry_list[entry]; 2554 npc_mcam_set_bit(mcam, index); 2555 mcam->entry2pfvf_map[index] = pcifunc; 2556 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2557 } 2558 2559 /* Update available free count in mbox response */ 2560 rsp->free_count = mcam->bmap_fcnt; 2561 2562 mutex_unlock(&mcam->lock); 2563 return 0; 2564 } 2565 2566 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2567 struct npc_mcam_alloc_entry_req *req, 2568 struct npc_mcam_alloc_entry_rsp *rsp) 2569 { 2570 struct npc_mcam *mcam = &rvu->hw->mcam; 2571 u16 pcifunc = req->hdr.pcifunc; 2572 int blkaddr; 2573 2574 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2575 if (blkaddr < 0) 2576 return NPC_MCAM_INVALID_REQ; 2577 2578 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2579 rsp->free_count = 0; 2580 2581 /* Check if ref_entry is within range */ 2582 if (req->priority && req->ref_entry >= mcam->bmap_entries) { 2583 dev_err(rvu->dev, "%s: reference entry %d is out of range\n", 2584 __func__, req->ref_entry); 2585 return NPC_MCAM_INVALID_REQ; 2586 } 2587 2588 /* ref_entry can't be '0' if requested priority is high. 2589 * Can't be last entry if requested priority is low. 2590 */ 2591 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2592 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2593 req->priority == NPC_MCAM_LOWER_PRIO)) 2594 return NPC_MCAM_INVALID_REQ; 2595 2596 /* Since list of allocated indices needs to be sent to requester, 2597 * max number of non-contiguous entries per mbox msg is limited. 2598 */ 2599 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 2600 dev_err(rvu->dev, 2601 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 2602 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 2603 return NPC_MCAM_INVALID_REQ; 2604 } 2605 2606 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2607 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2608 return NPC_MCAM_ALLOC_DENIED; 2609 2610 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2611 } 2612 2613 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2614 struct npc_mcam_free_entry_req *req, 2615 struct msg_rsp *rsp) 2616 { 2617 struct npc_mcam *mcam = &rvu->hw->mcam; 2618 u16 pcifunc = req->hdr.pcifunc; 2619 int blkaddr, rc = 0; 2620 u16 cntr; 2621 2622 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2623 if (blkaddr < 0) 2624 return NPC_MCAM_INVALID_REQ; 2625 2626 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2627 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2628 return NPC_MCAM_INVALID_REQ; 2629 2630 mutex_lock(&mcam->lock); 2631 2632 if (req->all) 2633 goto free_all; 2634 2635 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2636 if (rc) 2637 goto exit; 2638 2639 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP; 2640 mcam->entry2target_pffunc[req->entry] = 0x0; 2641 npc_mcam_clear_bit(mcam, req->entry); 2642 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2643 2644 /* Update entry2counter mapping */ 2645 cntr = mcam->entry2cntr_map[req->entry]; 2646 if (cntr != NPC_MCAM_INVALID_MAP) 2647 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2648 req->entry, cntr); 2649 2650 goto exit; 2651 2652 free_all: 2653 /* Free up all entries allocated to requesting PFFUNC */ 2654 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2655 exit: 2656 mutex_unlock(&mcam->lock); 2657 return rc; 2658 } 2659 2660 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2661 struct npc_mcam_read_entry_req *req, 2662 struct npc_mcam_read_entry_rsp *rsp) 2663 { 2664 struct npc_mcam *mcam = &rvu->hw->mcam; 2665 u16 pcifunc = req->hdr.pcifunc; 2666 int blkaddr, rc; 2667 2668 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2669 if (blkaddr < 0) 2670 return NPC_MCAM_INVALID_REQ; 2671 2672 mutex_lock(&mcam->lock); 2673 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2674 if (!rc) { 2675 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2676 &rsp->entry_data, 2677 &rsp->intf, &rsp->enable); 2678 } 2679 2680 mutex_unlock(&mcam->lock); 2681 return rc; 2682 } 2683 2684 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2685 struct npc_mcam_write_entry_req *req, 2686 struct msg_rsp *rsp) 2687 { 2688 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2689 struct npc_mcam *mcam = &rvu->hw->mcam; 2690 u16 pcifunc = req->hdr.pcifunc; 2691 int blkaddr, rc; 2692 u8 nix_intf; 2693 2694 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2695 if (blkaddr < 0) 2696 return NPC_MCAM_INVALID_REQ; 2697 2698 mutex_lock(&mcam->lock); 2699 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2700 if (rc) 2701 goto exit; 2702 2703 if (req->set_cntr && 2704 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2705 rc = NPC_MCAM_INVALID_REQ; 2706 goto exit; 2707 } 2708 2709 if (!is_npc_interface_valid(rvu, req->intf)) { 2710 rc = NPC_MCAM_INVALID_REQ; 2711 goto exit; 2712 } 2713 2714 if (is_npc_intf_tx(req->intf)) 2715 nix_intf = pfvf->nix_tx_intf; 2716 else 2717 nix_intf = pfvf->nix_rx_intf; 2718 2719 if (!is_pffunc_af(pcifunc) && 2720 npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, pcifunc)) { 2721 rc = NPC_MCAM_INVALID_REQ; 2722 goto exit; 2723 } 2724 2725 /* For AF installed rules, the nix_intf should be set to target NIX */ 2726 if (is_pffunc_af(req->hdr.pcifunc)) 2727 nix_intf = req->intf; 2728 2729 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2730 &req->entry_data, req->enable_entry); 2731 2732 if (req->set_cntr) 2733 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2734 req->entry, req->cntr); 2735 2736 rc = 0; 2737 exit: 2738 mutex_unlock(&mcam->lock); 2739 return rc; 2740 } 2741 2742 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2743 struct npc_mcam_ena_dis_entry_req *req, 2744 struct msg_rsp *rsp) 2745 { 2746 struct npc_mcam *mcam = &rvu->hw->mcam; 2747 u16 pcifunc = req->hdr.pcifunc; 2748 int blkaddr, rc; 2749 2750 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2751 if (blkaddr < 0) 2752 return NPC_MCAM_INVALID_REQ; 2753 2754 mutex_lock(&mcam->lock); 2755 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2756 mutex_unlock(&mcam->lock); 2757 if (rc) 2758 return rc; 2759 2760 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2761 2762 return 0; 2763 } 2764 2765 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2766 struct npc_mcam_ena_dis_entry_req *req, 2767 struct msg_rsp *rsp) 2768 { 2769 struct npc_mcam *mcam = &rvu->hw->mcam; 2770 u16 pcifunc = req->hdr.pcifunc; 2771 int blkaddr, rc; 2772 2773 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2774 if (blkaddr < 0) 2775 return NPC_MCAM_INVALID_REQ; 2776 2777 mutex_lock(&mcam->lock); 2778 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2779 mutex_unlock(&mcam->lock); 2780 if (rc) 2781 return rc; 2782 2783 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2784 2785 return 0; 2786 } 2787 2788 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2789 struct npc_mcam_shift_entry_req *req, 2790 struct npc_mcam_shift_entry_rsp *rsp) 2791 { 2792 struct npc_mcam *mcam = &rvu->hw->mcam; 2793 u16 pcifunc = req->hdr.pcifunc; 2794 u16 old_entry, new_entry; 2795 int blkaddr, rc = 0; 2796 u16 index, cntr; 2797 2798 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2799 if (blkaddr < 0) 2800 return NPC_MCAM_INVALID_REQ; 2801 2802 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2803 return NPC_MCAM_INVALID_REQ; 2804 2805 mutex_lock(&mcam->lock); 2806 for (index = 0; index < req->shift_count; index++) { 2807 old_entry = req->curr_entry[index]; 2808 new_entry = req->new_entry[index]; 2809 2810 /* Check if both old and new entries are valid and 2811 * does belong to this PFFUNC or not. 2812 */ 2813 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2814 if (rc) 2815 break; 2816 2817 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2818 if (rc) 2819 break; 2820 2821 /* new_entry should not have a counter mapped */ 2822 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2823 rc = NPC_MCAM_PERM_DENIED; 2824 break; 2825 } 2826 2827 /* Disable the new_entry */ 2828 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2829 2830 /* Copy rule from old entry to new entry */ 2831 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2832 2833 /* Copy counter mapping, if any */ 2834 cntr = mcam->entry2cntr_map[old_entry]; 2835 if (cntr != NPC_MCAM_INVALID_MAP) { 2836 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2837 old_entry, cntr); 2838 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2839 new_entry, cntr); 2840 } 2841 2842 /* Enable new_entry and disable old_entry */ 2843 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2844 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2845 } 2846 2847 /* If shift has failed then report the failed index */ 2848 if (index != req->shift_count) { 2849 rc = NPC_MCAM_PERM_DENIED; 2850 rsp->failed_entry_idx = index; 2851 } 2852 2853 mutex_unlock(&mcam->lock); 2854 return rc; 2855 } 2856 2857 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2858 struct npc_mcam_alloc_counter_req *req, 2859 struct npc_mcam_alloc_counter_rsp *rsp) 2860 { 2861 struct npc_mcam *mcam = &rvu->hw->mcam; 2862 u16 pcifunc = req->hdr.pcifunc; 2863 u16 max_contig, cntr; 2864 int blkaddr, index; 2865 2866 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2867 if (blkaddr < 0) 2868 return NPC_MCAM_INVALID_REQ; 2869 2870 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2871 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc)) 2872 return NPC_MCAM_INVALID_REQ; 2873 2874 /* Since list of allocated counter IDs needs to be sent to requester, 2875 * max number of non-contiguous counters per mbox msg is limited. 2876 */ 2877 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2878 return NPC_MCAM_INVALID_REQ; 2879 2880 mutex_lock(&mcam->lock); 2881 2882 /* Check if unused counters are available or not */ 2883 if (!rvu_rsrc_free_count(&mcam->counters)) { 2884 mutex_unlock(&mcam->lock); 2885 return NPC_MCAM_ALLOC_FAILED; 2886 } 2887 2888 rsp->count = 0; 2889 2890 if (req->contig) { 2891 /* Allocate requested number of contiguous counters, if 2892 * unsuccessful find max contiguous entries available. 2893 */ 2894 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2895 mcam->counters.max, 0, 2896 req->count, &max_contig); 2897 rsp->count = max_contig; 2898 rsp->cntr = index; 2899 for (cntr = index; cntr < (index + max_contig); cntr++) { 2900 __set_bit(cntr, mcam->counters.bmap); 2901 mcam->cntr2pfvf_map[cntr] = pcifunc; 2902 } 2903 } else { 2904 /* Allocate requested number of non-contiguous counters, 2905 * if unsuccessful allocate as many as possible. 2906 */ 2907 for (cntr = 0; cntr < req->count; cntr++) { 2908 index = rvu_alloc_rsrc(&mcam->counters); 2909 if (index < 0) 2910 break; 2911 rsp->cntr_list[cntr] = index; 2912 rsp->count++; 2913 mcam->cntr2pfvf_map[index] = pcifunc; 2914 } 2915 } 2916 2917 mutex_unlock(&mcam->lock); 2918 return 0; 2919 } 2920 2921 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2922 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2923 { 2924 struct npc_mcam *mcam = &rvu->hw->mcam; 2925 u16 index, entry = 0; 2926 int blkaddr, err; 2927 2928 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2929 if (blkaddr < 0) 2930 return NPC_MCAM_INVALID_REQ; 2931 2932 mutex_lock(&mcam->lock); 2933 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2934 if (err) { 2935 mutex_unlock(&mcam->lock); 2936 return err; 2937 } 2938 2939 /* Mark counter as free/unused */ 2940 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2941 rvu_free_rsrc(&mcam->counters, req->cntr); 2942 2943 /* Disable all MCAM entry's stats which are using this counter */ 2944 while (entry < mcam->bmap_entries) { 2945 if (!mcam->cntr_refcnt[req->cntr]) 2946 break; 2947 2948 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2949 if (index >= mcam->bmap_entries) 2950 break; 2951 entry = index + 1; 2952 if (mcam->entry2cntr_map[index] != req->cntr) 2953 continue; 2954 2955 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2956 index, req->cntr); 2957 } 2958 2959 mutex_unlock(&mcam->lock); 2960 return 0; 2961 } 2962 2963 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2964 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2965 { 2966 struct npc_mcam *mcam = &rvu->hw->mcam; 2967 u16 index, entry = 0; 2968 int blkaddr, rc; 2969 2970 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2971 if (blkaddr < 0) 2972 return NPC_MCAM_INVALID_REQ; 2973 2974 mutex_lock(&mcam->lock); 2975 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2976 if (rc) 2977 goto exit; 2978 2979 /* Unmap the MCAM entry and counter */ 2980 if (!req->all) { 2981 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2982 if (rc) 2983 goto exit; 2984 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2985 req->entry, req->cntr); 2986 goto exit; 2987 } 2988 2989 /* Disable all MCAM entry's stats which are using this counter */ 2990 while (entry < mcam->bmap_entries) { 2991 if (!mcam->cntr_refcnt[req->cntr]) 2992 break; 2993 2994 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2995 if (index >= mcam->bmap_entries) 2996 break; 2997 entry = index + 1; 2998 2999 if (mcam->entry2cntr_map[index] != req->cntr) 3000 continue; 3001 3002 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 3003 index, req->cntr); 3004 } 3005 exit: 3006 mutex_unlock(&mcam->lock); 3007 return rc; 3008 } 3009 3010 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 3011 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 3012 { 3013 struct npc_mcam *mcam = &rvu->hw->mcam; 3014 int blkaddr, err; 3015 3016 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3017 if (blkaddr < 0) 3018 return NPC_MCAM_INVALID_REQ; 3019 3020 mutex_lock(&mcam->lock); 3021 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3022 mutex_unlock(&mcam->lock); 3023 if (err) 3024 return err; 3025 3026 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 3027 3028 return 0; 3029 } 3030 3031 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 3032 struct npc_mcam_oper_counter_req *req, 3033 struct npc_mcam_oper_counter_rsp *rsp) 3034 { 3035 struct npc_mcam *mcam = &rvu->hw->mcam; 3036 int blkaddr, err; 3037 3038 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3039 if (blkaddr < 0) 3040 return NPC_MCAM_INVALID_REQ; 3041 3042 mutex_lock(&mcam->lock); 3043 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3044 mutex_unlock(&mcam->lock); 3045 if (err) 3046 return err; 3047 3048 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3049 rsp->stat &= BIT_ULL(48) - 1; 3050 3051 return 0; 3052 } 3053 3054 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3055 struct npc_mcam_alloc_and_write_entry_req *req, 3056 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3057 { 3058 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3059 struct npc_mcam_alloc_counter_req cntr_req; 3060 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3061 struct npc_mcam_alloc_entry_req entry_req; 3062 struct npc_mcam_alloc_entry_rsp entry_rsp; 3063 struct npc_mcam *mcam = &rvu->hw->mcam; 3064 u16 entry = NPC_MCAM_ENTRY_INVALID; 3065 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3066 int blkaddr, rc; 3067 u8 nix_intf; 3068 3069 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3070 if (blkaddr < 0) 3071 return NPC_MCAM_INVALID_REQ; 3072 3073 if (!is_npc_interface_valid(rvu, req->intf)) 3074 return NPC_MCAM_INVALID_REQ; 3075 3076 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 3077 req->hdr.pcifunc)) 3078 return NPC_MCAM_INVALID_REQ; 3079 3080 /* Try to allocate a MCAM entry */ 3081 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3082 entry_req.contig = true; 3083 entry_req.priority = req->priority; 3084 entry_req.ref_entry = req->ref_entry; 3085 entry_req.count = 1; 3086 3087 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3088 &entry_req, &entry_rsp); 3089 if (rc) 3090 return rc; 3091 3092 if (!entry_rsp.count) 3093 return NPC_MCAM_ALLOC_FAILED; 3094 3095 entry = entry_rsp.entry; 3096 3097 if (!req->alloc_cntr) 3098 goto write_entry; 3099 3100 /* Now allocate counter */ 3101 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3102 cntr_req.contig = true; 3103 cntr_req.count = 1; 3104 3105 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3106 if (rc) { 3107 /* Free allocated MCAM entry */ 3108 mutex_lock(&mcam->lock); 3109 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP; 3110 npc_mcam_clear_bit(mcam, entry); 3111 mutex_unlock(&mcam->lock); 3112 return rc; 3113 } 3114 3115 cntr = cntr_rsp.cntr; 3116 3117 write_entry: 3118 mutex_lock(&mcam->lock); 3119 3120 if (is_npc_intf_tx(req->intf)) 3121 nix_intf = pfvf->nix_tx_intf; 3122 else 3123 nix_intf = pfvf->nix_rx_intf; 3124 3125 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3126 &req->entry_data, req->enable_entry); 3127 3128 if (req->alloc_cntr) 3129 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3130 mutex_unlock(&mcam->lock); 3131 3132 rsp->entry = entry; 3133 rsp->cntr = cntr; 3134 3135 return 0; 3136 } 3137 3138 #define GET_KEX_CFG(intf) \ 3139 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3140 3141 #define GET_KEX_FLAGS(ld) \ 3142 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3143 3144 #define GET_KEX_LD(intf, lid, lt, ld) \ 3145 rvu_read64(rvu, BLKADDR_NPC, \ 3146 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3147 3148 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3149 rvu_read64(rvu, BLKADDR_NPC, \ 3150 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3151 3152 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3153 struct npc_get_kex_cfg_rsp *rsp) 3154 { 3155 int lid, lt, ld, fl; 3156 3157 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3158 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3159 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3160 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3161 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3162 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3163 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3164 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3165 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3166 } 3167 } 3168 } 3169 for (ld = 0; ld < NPC_MAX_LD; ld++) 3170 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3171 3172 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3173 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3174 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3175 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3176 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3177 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3178 } 3179 } 3180 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3181 return 0; 3182 } 3183 3184 static int 3185 npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind, 3186 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir) 3187 { 3188 struct npc_kpu_action0 *act0; 3189 u8 shift_count = 0; 3190 int blkaddr; 3191 u64 val; 3192 3193 if (!var_len_off_mask) 3194 return -EINVAL; 3195 3196 if (var_len_off_mask != 0xff) { 3197 if (shift_dir) 3198 shift_count = __ffs(var_len_off_mask); 3199 else 3200 shift_count = (8 - __fls(var_len_off_mask)); 3201 } 3202 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 3203 if (blkaddr < 0) { 3204 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 3205 return -EINVAL; 3206 } 3207 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 3208 act0 = (struct npc_kpu_action0 *)&val; 3209 act0->var_len_shift = shift_count; 3210 act0->var_len_right = shift_dir; 3211 act0->var_len_mask = var_len_off_mask; 3212 act0->var_len_offset = var_len_off; 3213 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 3214 return 0; 3215 } 3216 3217 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir, 3218 u64 pkind, u8 var_len_off, u8 var_len_off_mask, 3219 u8 shift_dir) 3220 3221 { 3222 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 3223 int blkaddr, nixlf, rc, intf_mode; 3224 int pf = rvu_get_pf(pcifunc); 3225 u64 rxpkind, txpkind; 3226 u8 cgx_id, lmac_id; 3227 3228 /* use default pkind to disable edsa/higig */ 3229 rxpkind = rvu_npc_get_pkind(rvu, pf); 3230 txpkind = NPC_TX_DEF_PKIND; 3231 intf_mode = NPC_INTF_MODE_DEF; 3232 3233 if (mode & OTX2_PRIV_FLAGS_CUSTOM) { 3234 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) { 3235 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind, 3236 var_len_off, 3237 var_len_off_mask, 3238 shift_dir); 3239 if (rc) 3240 return rc; 3241 } 3242 rxpkind = pkind; 3243 txpkind = pkind; 3244 } 3245 3246 if (dir & PKIND_RX) { 3247 /* rx pkind set req valid only for cgx mapped PFs */ 3248 if (!is_cgx_config_permitted(rvu, pcifunc)) 3249 return 0; 3250 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3251 3252 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, 3253 rxpkind); 3254 if (rc) 3255 return rc; 3256 } 3257 3258 if (dir & PKIND_TX) { 3259 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */ 3260 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3261 if (rc) 3262 return rc; 3263 3264 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), 3265 txpkind); 3266 } 3267 3268 pfvf->intf_mode = intf_mode; 3269 return 0; 3270 } 3271 3272 int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req, 3273 struct msg_rsp *rsp) 3274 { 3275 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode, 3276 req->dir, req->pkind, req->var_len_off, 3277 req->var_len_off_mask, req->shift_dir); 3278 } 3279 3280 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3281 struct msg_req *req, 3282 struct npc_mcam_read_base_rule_rsp *rsp) 3283 { 3284 struct npc_mcam *mcam = &rvu->hw->mcam; 3285 int index, blkaddr, nixlf, rc = 0; 3286 u16 pcifunc = req->hdr.pcifunc; 3287 struct rvu_pfvf *pfvf; 3288 u8 intf, enable; 3289 3290 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3291 if (blkaddr < 0) 3292 return NPC_MCAM_INVALID_REQ; 3293 3294 /* Return the channel number in case of PF */ 3295 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3296 pfvf = rvu_get_pfvf(rvu, pcifunc); 3297 rsp->entry.kw[0] = pfvf->rx_chan_base; 3298 rsp->entry.kw_mask[0] = 0xFFFULL; 3299 goto out; 3300 } 3301 3302 /* Find the pkt steering rule installed by PF to this VF */ 3303 mutex_lock(&mcam->lock); 3304 for (index = 0; index < mcam->bmap_entries; index++) { 3305 if (mcam->entry2target_pffunc[index] == pcifunc) 3306 goto read_entry; 3307 } 3308 3309 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3310 if (rc < 0) { 3311 mutex_unlock(&mcam->lock); 3312 goto out; 3313 } 3314 /* Read the default ucast entry if there is no pkt steering rule */ 3315 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3316 NIXLF_UCAST_ENTRY); 3317 read_entry: 3318 /* Read the mcam entry */ 3319 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3320 &enable); 3321 mutex_unlock(&mcam->lock); 3322 out: 3323 return rc; 3324 } 3325 3326 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3327 struct npc_mcam_get_stats_req *req, 3328 struct npc_mcam_get_stats_rsp *rsp) 3329 { 3330 struct npc_mcam *mcam = &rvu->hw->mcam; 3331 u16 index, cntr; 3332 int blkaddr; 3333 u64 regval; 3334 u32 bank; 3335 3336 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3337 if (blkaddr < 0) 3338 return NPC_MCAM_INVALID_REQ; 3339 3340 mutex_lock(&mcam->lock); 3341 3342 index = req->entry & (mcam->banksize - 1); 3343 bank = npc_get_bank(mcam, req->entry); 3344 3345 /* read MCAM entry STAT_ACT register */ 3346 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3347 3348 if (!(regval & rvu->hw->npc_stat_ena)) { 3349 rsp->stat_ena = 0; 3350 mutex_unlock(&mcam->lock); 3351 return 0; 3352 } 3353 3354 cntr = regval & 0x1FF; 3355 3356 rsp->stat_ena = 1; 3357 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3358 rsp->stat &= BIT_ULL(48) - 1; 3359 3360 mutex_unlock(&mcam->lock); 3361 3362 return 0; 3363 } 3364