1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "cgx.h" 20 #include "npc_profile.h" 21 22 #define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ 23 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 24 25 #define NIXLF_UCAST_ENTRY 0 26 #define NIXLF_BCAST_ENTRY 1 27 #define NIXLF_PROMISC_ENTRY 2 28 29 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 30 #define NPC_HW_TSTAMP_OFFSET 8 31 32 static const char def_pfl_name[] = "default"; 33 34 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 35 int blkaddr, u16 pcifunc); 36 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 37 u16 pcifunc); 38 39 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 40 { 41 int blkaddr; 42 u64 val = 0; 43 44 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 45 if (blkaddr < 0) 46 return; 47 48 /* Config CPI base for the PKIND */ 49 val = pkind | 1ULL << 62; 50 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 51 } 52 53 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 54 { 55 struct npc_pkind *pkind = &rvu->hw->pkind; 56 u32 map; 57 int i; 58 59 for (i = 0; i < pkind->rsrc.max; i++) { 60 map = pkind->pfchan_map[i]; 61 if (((map >> 16) & 0x3F) == pf) 62 return i; 63 } 64 return -1; 65 } 66 67 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 68 69 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 70 { 71 int pkind, blkaddr; 72 u64 val; 73 74 pkind = rvu_npc_get_pkind(rvu, pf); 75 if (pkind < 0) { 76 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 77 return -EINVAL; 78 } 79 80 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 81 if (blkaddr < 0) { 82 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 83 return -EINVAL; 84 } 85 86 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 87 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 88 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 89 if (enable) 90 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 91 NPC_HW_TSTAMP_OFFSET); 92 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 93 94 return 0; 95 } 96 97 static int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 98 u16 pcifunc, int nixlf, int type) 99 { 100 int pf = rvu_get_pf(pcifunc); 101 int index; 102 103 /* Check if this is for a PF */ 104 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 105 /* Reserved entries exclude PF0 */ 106 pf--; 107 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 108 /* Broadcast address matching entry should be first so 109 * that the packet can be replicated to all VFs. 110 */ 111 if (type == NIXLF_BCAST_ENTRY) 112 return index; 113 else if (type == NIXLF_PROMISC_ENTRY) 114 return index + 1; 115 } 116 117 return (mcam->nixlf_offset + (nixlf * RSVD_MCAM_ENTRIES_PER_NIXLF)); 118 } 119 120 static int npc_get_bank(struct npc_mcam *mcam, int index) 121 { 122 int bank = index / mcam->banksize; 123 124 /* 0,1 & 2,3 banks are combined for this keysize */ 125 if (mcam->keysize == NPC_MCAM_KEY_X2) 126 return bank ? 2 : 0; 127 128 return bank; 129 } 130 131 static bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 132 int blkaddr, int index) 133 { 134 int bank = npc_get_bank(mcam, index); 135 u64 cfg; 136 137 index &= (mcam->banksize - 1); 138 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 139 return (cfg & 1); 140 } 141 142 static void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 143 int blkaddr, int index, bool enable) 144 { 145 int bank = npc_get_bank(mcam, index); 146 int actbank = bank; 147 148 index &= (mcam->banksize - 1); 149 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 150 rvu_write64(rvu, blkaddr, 151 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 152 enable ? 1 : 0); 153 } 154 } 155 156 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 157 int blkaddr, int index) 158 { 159 int bank = npc_get_bank(mcam, index); 160 int actbank = bank; 161 162 index &= (mcam->banksize - 1); 163 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 164 rvu_write64(rvu, blkaddr, 165 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 166 rvu_write64(rvu, blkaddr, 167 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 168 169 rvu_write64(rvu, blkaddr, 170 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 171 rvu_write64(rvu, blkaddr, 172 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 173 174 rvu_write64(rvu, blkaddr, 175 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 176 rvu_write64(rvu, blkaddr, 177 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 178 } 179 } 180 181 static void npc_get_keyword(struct mcam_entry *entry, int idx, 182 u64 *cam0, u64 *cam1) 183 { 184 u64 kw_mask = 0x00; 185 186 #define CAM_MASK(n) (BIT_ULL(n) - 1) 187 188 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 189 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 190 * 191 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 192 */ 193 switch (idx) { 194 case 0: 195 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 196 *cam1 = entry->kw[0]; 197 kw_mask = entry->kw_mask[0]; 198 break; 199 case 1: 200 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 201 *cam1 = entry->kw[1] & CAM_MASK(48); 202 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 203 break; 204 case 2: 205 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 206 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 207 */ 208 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 209 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 210 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 211 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 212 break; 213 case 3: 214 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 215 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 216 */ 217 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 218 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 219 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 220 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 221 break; 222 case 4: 223 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 224 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 225 */ 226 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 227 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 228 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 229 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 230 break; 231 case 5: 232 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 233 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 234 */ 235 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 236 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 237 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 238 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 239 break; 240 case 6: 241 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 242 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 243 */ 244 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 245 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 246 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 247 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 248 break; 249 case 7: 250 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 251 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 252 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 253 break; 254 } 255 256 *cam1 &= kw_mask; 257 *cam0 = ~*cam1 & kw_mask; 258 } 259 260 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 261 int blkaddr, int index, u8 intf, 262 struct mcam_entry *entry, bool enable) 263 { 264 int bank = npc_get_bank(mcam, index); 265 int kw = 0, actbank, actindex; 266 u64 cam0, cam1; 267 268 actbank = bank; /* Save bank id, to set action later on */ 269 actindex = index; 270 index &= (mcam->banksize - 1); 271 272 /* Disable before mcam entry update */ 273 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 274 275 /* Clear mcam entry to avoid writes being suppressed by NPC */ 276 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 277 278 /* CAM1 takes the comparison value and 279 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 280 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 281 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 282 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 283 */ 284 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 285 /* Interface should be set in all banks */ 286 rvu_write64(rvu, blkaddr, 287 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 288 intf); 289 rvu_write64(rvu, blkaddr, 290 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 291 ~intf & 0x3); 292 293 /* Set the match key */ 294 npc_get_keyword(entry, kw, &cam0, &cam1); 295 rvu_write64(rvu, blkaddr, 296 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 297 rvu_write64(rvu, blkaddr, 298 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 299 300 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 301 rvu_write64(rvu, blkaddr, 302 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 303 rvu_write64(rvu, blkaddr, 304 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 305 } 306 307 /* Set 'action' */ 308 rvu_write64(rvu, blkaddr, 309 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 310 311 /* Set TAG 'action' */ 312 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 313 entry->vtag_action); 314 315 /* Enable the entry */ 316 if (enable) 317 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 318 } 319 320 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 321 int blkaddr, u16 src, u16 dest) 322 { 323 int dbank = npc_get_bank(mcam, dest); 324 int sbank = npc_get_bank(mcam, src); 325 u64 cfg, sreg, dreg; 326 int bank, i; 327 328 src &= (mcam->banksize - 1); 329 dest &= (mcam->banksize - 1); 330 331 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 332 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 333 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 334 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 335 for (i = 0; i < 6; i++) { 336 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 337 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 338 } 339 } 340 341 /* Copy action */ 342 cfg = rvu_read64(rvu, blkaddr, 343 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 344 rvu_write64(rvu, blkaddr, 345 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 346 347 /* Copy TAG action */ 348 cfg = rvu_read64(rvu, blkaddr, 349 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 350 rvu_write64(rvu, blkaddr, 351 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 352 353 /* Enable or disable */ 354 cfg = rvu_read64(rvu, blkaddr, 355 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 356 rvu_write64(rvu, blkaddr, 357 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 358 } 359 360 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 361 int blkaddr, int index) 362 { 363 int bank = npc_get_bank(mcam, index); 364 365 index &= (mcam->banksize - 1); 366 return rvu_read64(rvu, blkaddr, 367 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 368 } 369 370 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 371 int nixlf, u64 chan, u8 *mac_addr) 372 { 373 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 374 struct npc_mcam *mcam = &rvu->hw->mcam; 375 struct mcam_entry entry = { {0} }; 376 struct nix_rx_action action; 377 int blkaddr, index, kwi; 378 u64 mac = 0; 379 380 /* AF's VFs work in promiscuous mode */ 381 if (is_afvf(pcifunc)) 382 return; 383 384 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 385 if (blkaddr < 0) 386 return; 387 388 for (index = ETH_ALEN - 1; index >= 0; index--) 389 mac |= ((u64)*mac_addr++) << (8 * index); 390 391 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 392 nixlf, NIXLF_UCAST_ENTRY); 393 394 /* Match ingress channel and DMAC */ 395 entry.kw[0] = chan; 396 entry.kw_mask[0] = 0xFFFULL; 397 398 kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); 399 entry.kw[kwi] = mac; 400 entry.kw_mask[kwi] = BIT_ULL(48) - 1; 401 402 /* Don't change the action if entry is already enabled 403 * Otherwise RSS action may get overwritten. 404 */ 405 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 406 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 407 blkaddr, index); 408 } else { 409 *(u64 *)&action = 0x00; 410 action.op = NIX_RX_ACTIONOP_UCAST; 411 action.pf_func = pcifunc; 412 } 413 414 entry.action = *(u64 *)&action; 415 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 416 NIX_INTF_RX, &entry, true); 417 418 /* add VLAN matching, setup action and save entry back for later */ 419 entry.kw[0] |= (NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG) << 20; 420 entry.kw_mask[0] |= (NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG) << 20; 421 422 entry.vtag_action = VTAG0_VALID_BIT | 423 FIELD_PREP(VTAG0_TYPE_MASK, 0) | 424 FIELD_PREP(VTAG0_LID_MASK, NPC_LID_LA) | 425 FIELD_PREP(VTAG0_RELPTR_MASK, 12); 426 427 memcpy(&pfvf->entry, &entry, sizeof(entry)); 428 } 429 430 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 431 int nixlf, u64 chan, bool allmulti) 432 { 433 struct npc_mcam *mcam = &rvu->hw->mcam; 434 int blkaddr, ucast_idx, index, kwi; 435 struct mcam_entry entry = { {0} }; 436 struct nix_rx_action action = { }; 437 438 /* Only PF or AF VF can add a promiscuous entry */ 439 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) 440 return; 441 442 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 443 if (blkaddr < 0) 444 return; 445 446 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 447 nixlf, NIXLF_PROMISC_ENTRY); 448 449 entry.kw[0] = chan; 450 entry.kw_mask[0] = 0xFFFULL; 451 452 if (allmulti) { 453 kwi = NPC_KEXOF_DMAC / sizeof(u64); 454 entry.kw[kwi] = BIT_ULL(40); /* LSB bit of 1st byte in DMAC */ 455 entry.kw_mask[kwi] = BIT_ULL(40); 456 } 457 458 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 459 nixlf, NIXLF_UCAST_ENTRY); 460 461 /* If the corresponding PF's ucast action is RSS, 462 * use the same action for promisc also 463 */ 464 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 465 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 466 blkaddr, ucast_idx); 467 468 if (action.op != NIX_RX_ACTIONOP_RSS) { 469 *(u64 *)&action = 0x00; 470 action.op = NIX_RX_ACTIONOP_UCAST; 471 action.pf_func = pcifunc; 472 } 473 474 entry.action = *(u64 *)&action; 475 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 476 NIX_INTF_RX, &entry, true); 477 } 478 479 static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, 480 int nixlf, bool enable) 481 { 482 struct npc_mcam *mcam = &rvu->hw->mcam; 483 int blkaddr, index; 484 485 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 486 if (blkaddr < 0) 487 return; 488 489 /* Only PF's have a promiscuous entry */ 490 if (pcifunc & RVU_PFVF_FUNC_MASK) 491 return; 492 493 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 494 nixlf, NIXLF_PROMISC_ENTRY); 495 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 496 } 497 498 void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 499 { 500 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); 501 } 502 503 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 504 { 505 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); 506 } 507 508 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 509 int nixlf, u64 chan) 510 { 511 struct npc_mcam *mcam = &rvu->hw->mcam; 512 struct mcam_entry entry = { {0} }; 513 struct rvu_hwinfo *hw = rvu->hw; 514 struct nix_rx_action action; 515 struct rvu_pfvf *pfvf; 516 int blkaddr, index; 517 518 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 519 if (blkaddr < 0) 520 return; 521 522 /* Skip LBK VFs */ 523 if (is_afvf(pcifunc)) 524 return; 525 526 /* If pkt replication is not supported, 527 * then only PF is allowed to add a bcast match entry. 528 */ 529 if (!hw->cap.nix_rx_multicast && pcifunc & RVU_PFVF_FUNC_MASK) 530 return; 531 532 /* Get 'pcifunc' of PF device */ 533 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 534 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 535 nixlf, NIXLF_BCAST_ENTRY); 536 537 /* Match ingress channel */ 538 entry.kw[0] = chan; 539 entry.kw_mask[0] = 0xfffull; 540 541 /* Match broadcast MAC address. 542 * DMAC is extracted at 0th bit of PARSE_KEX::KW1 543 */ 544 entry.kw[1] = 0xffffffffffffull; 545 entry.kw_mask[1] = 0xffffffffffffull; 546 547 *(u64 *)&action = 0x00; 548 if (!hw->cap.nix_rx_multicast) { 549 /* Early silicon doesn't support pkt replication, 550 * so install entry with UCAST action, so that PF 551 * receives all broadcast packets. 552 */ 553 action.op = NIX_RX_ACTIONOP_UCAST; 554 action.pf_func = pcifunc; 555 } else { 556 pfvf = rvu_get_pfvf(rvu, pcifunc); 557 action.index = pfvf->bcast_mce_idx; 558 action.op = NIX_RX_ACTIONOP_MCAST; 559 } 560 561 entry.action = *(u64 *)&action; 562 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 563 NIX_INTF_RX, &entry, true); 564 } 565 566 void rvu_npc_disable_bcast_entry(struct rvu *rvu, u16 pcifunc) 567 { 568 struct npc_mcam *mcam = &rvu->hw->mcam; 569 int blkaddr, index; 570 571 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 572 if (blkaddr < 0) 573 return; 574 575 /* Get 'pcifunc' of PF device */ 576 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 577 578 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 0, NIXLF_BCAST_ENTRY); 579 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 580 } 581 582 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 583 int group, int alg_idx, int mcam_index) 584 { 585 struct npc_mcam *mcam = &rvu->hw->mcam; 586 struct nix_rx_action action; 587 int blkaddr, index, bank; 588 589 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 590 if (blkaddr < 0) 591 return; 592 593 /* Check if this is for reserved default entry */ 594 if (mcam_index < 0) { 595 if (group != DEFAULT_RSS_CONTEXT_GROUP) 596 return; 597 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 598 nixlf, NIXLF_UCAST_ENTRY); 599 } else { 600 /* TODO: validate this mcam index */ 601 index = mcam_index; 602 } 603 604 if (index >= mcam->total_entries) 605 return; 606 607 bank = npc_get_bank(mcam, index); 608 index &= (mcam->banksize - 1); 609 610 *(u64 *)&action = rvu_read64(rvu, blkaddr, 611 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 612 /* Ignore if no action was set earlier */ 613 if (!*(u64 *)&action) 614 return; 615 616 action.op = NIX_RX_ACTIONOP_RSS; 617 action.pf_func = pcifunc; 618 action.index = group; 619 action.flow_key_alg = alg_idx; 620 621 rvu_write64(rvu, blkaddr, 622 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 623 624 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 625 nixlf, NIXLF_PROMISC_ENTRY); 626 627 /* If PF's promiscuous entry is enabled, 628 * Set RSS action for that entry as well 629 */ 630 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 631 bank = npc_get_bank(mcam, index); 632 index &= (mcam->banksize - 1); 633 634 rvu_write64(rvu, blkaddr, 635 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 636 *(u64 *)&action); 637 } 638 639 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 640 } 641 642 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 643 int nixlf, bool enable) 644 { 645 struct npc_mcam *mcam = &rvu->hw->mcam; 646 struct nix_rx_action action; 647 int index, bank, blkaddr; 648 649 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 650 if (blkaddr < 0) 651 return; 652 653 /* Ucast MCAM match entry of this PF/VF */ 654 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 655 nixlf, NIXLF_UCAST_ENTRY); 656 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 657 658 /* For PF, ena/dis promisc and bcast MCAM match entries */ 659 if (pcifunc & RVU_PFVF_FUNC_MASK) 660 return; 661 662 /* For bcast, enable/disable only if it's action is not 663 * packet replication, incase if action is replication 664 * then this PF's nixlf is removed from bcast replication 665 * list. 666 */ 667 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 668 nixlf, NIXLF_BCAST_ENTRY); 669 bank = npc_get_bank(mcam, index); 670 *(u64 *)&action = rvu_read64(rvu, blkaddr, 671 NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); 672 if (action.op != NIX_RX_ACTIONOP_MCAST) 673 npc_enable_mcam_entry(rvu, mcam, 674 blkaddr, index, enable); 675 if (enable) 676 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); 677 else 678 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); 679 680 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 681 } 682 683 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 684 { 685 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 686 } 687 688 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 689 { 690 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 691 } 692 693 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 694 { 695 struct npc_mcam *mcam = &rvu->hw->mcam; 696 int blkaddr; 697 698 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 699 if (blkaddr < 0) 700 return; 701 702 mutex_lock(&mcam->lock); 703 704 /* Disable and free all MCAM entries mapped to this 'pcifunc' */ 705 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 706 707 /* Free all MCAM counters mapped to this 'pcifunc' */ 708 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 709 710 mutex_unlock(&mcam->lock); 711 712 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 713 } 714 715 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 716 rvu_write64(rvu, blkaddr, \ 717 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 718 719 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 720 rvu_write64(rvu, blkaddr, \ 721 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 722 723 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 724 const struct npc_mcam_kex *mkex) 725 { 726 int lid, lt, ld, fl; 727 728 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), 729 mkex->keyx_cfg[NIX_INTF_RX]); 730 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), 731 mkex->keyx_cfg[NIX_INTF_TX]); 732 733 for (ld = 0; ld < NPC_MAX_LD; ld++) 734 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 735 mkex->kex_ld_flags[ld]); 736 737 for (lid = 0; lid < NPC_MAX_LID; lid++) { 738 for (lt = 0; lt < NPC_MAX_LT; lt++) { 739 for (ld = 0; ld < NPC_MAX_LD; ld++) { 740 SET_KEX_LD(NIX_INTF_RX, lid, lt, ld, 741 mkex->intf_lid_lt_ld[NIX_INTF_RX] 742 [lid][lt][ld]); 743 744 SET_KEX_LD(NIX_INTF_TX, lid, lt, ld, 745 mkex->intf_lid_lt_ld[NIX_INTF_TX] 746 [lid][lt][ld]); 747 } 748 } 749 } 750 751 for (ld = 0; ld < NPC_MAX_LD; ld++) { 752 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 753 SET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl, 754 mkex->intf_ld_flags[NIX_INTF_RX] 755 [ld][fl]); 756 757 SET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl, 758 mkex->intf_ld_flags[NIX_INTF_TX] 759 [ld][fl]); 760 } 761 } 762 } 763 764 #define MKEX_END_SIGN 0xdeadbeef 765 766 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 767 const char *mkex_profile) 768 { 769 struct device *dev = &rvu->pdev->dev; 770 struct npc_mcam_kex *mcam_kex; 771 void *mkex_prfl_addr = NULL; 772 u64 prfl_addr, prfl_sz; 773 774 /* If user not selected mkex profile */ 775 if (!strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 776 goto program_mkex; 777 778 if (!rvu->fwdata) 779 goto program_mkex; 780 prfl_addr = rvu->fwdata->mcam_addr; 781 prfl_sz = rvu->fwdata->mcam_sz; 782 783 if (!prfl_addr || !prfl_sz) 784 goto program_mkex; 785 786 mkex_prfl_addr = memremap(prfl_addr, prfl_sz, MEMREMAP_WC); 787 if (!mkex_prfl_addr) 788 goto program_mkex; 789 790 mcam_kex = (struct npc_mcam_kex *)mkex_prfl_addr; 791 792 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 793 /* Compare with mkex mod_param name string */ 794 if (mcam_kex->mkex_sign == MKEX_SIGN && 795 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 796 /* Due to an errata (35786) in A0/B0 pass silicon, 797 * parse nibble enable configuration has to be 798 * identical for both Rx and Tx interfaces. 799 */ 800 if (!is_rvu_96xx_B0(rvu) || 801 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 802 rvu->kpu.mkex = mcam_kex; 803 goto program_mkex; 804 } 805 806 mcam_kex++; 807 prfl_sz -= sizeof(struct npc_mcam_kex); 808 } 809 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 810 811 program_mkex: 812 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 813 /* Program selected mkex profile */ 814 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 815 if (mkex_prfl_addr) 816 memunmap(mkex_prfl_addr); 817 } 818 819 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 820 const struct npc_kpu_profile_action *kpuaction, 821 int kpu, int entry, bool pkind) 822 { 823 struct npc_kpu_action0 action0 = {0}; 824 struct npc_kpu_action1 action1 = {0}; 825 u64 reg; 826 827 action1.errlev = kpuaction->errlev; 828 action1.errcode = kpuaction->errcode; 829 action1.dp0_offset = kpuaction->dp0_offset; 830 action1.dp1_offset = kpuaction->dp1_offset; 831 action1.dp2_offset = kpuaction->dp2_offset; 832 833 if (pkind) 834 reg = NPC_AF_PKINDX_ACTION1(entry); 835 else 836 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 837 838 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 839 840 action0.byp_count = kpuaction->bypass_count; 841 action0.capture_ena = kpuaction->cap_ena; 842 action0.parse_done = kpuaction->parse_done; 843 action0.next_state = kpuaction->next_state; 844 action0.capture_lid = kpuaction->lid; 845 action0.capture_ltype = kpuaction->ltype; 846 action0.capture_flags = kpuaction->flags; 847 action0.ptr_advance = kpuaction->ptr_advance; 848 action0.var_len_offset = kpuaction->offset; 849 action0.var_len_mask = kpuaction->mask; 850 action0.var_len_right = kpuaction->right; 851 action0.var_len_shift = kpuaction->shift; 852 853 if (pkind) 854 reg = NPC_AF_PKINDX_ACTION0(entry); 855 else 856 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 857 858 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 859 } 860 861 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 862 const struct npc_kpu_profile_cam *kpucam, 863 int kpu, int entry) 864 { 865 struct npc_kpu_cam cam0 = {0}; 866 struct npc_kpu_cam cam1 = {0}; 867 868 cam1.state = kpucam->state & kpucam->state_mask; 869 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 870 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 871 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 872 873 cam0.state = ~kpucam->state & kpucam->state_mask; 874 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 875 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 876 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 877 878 rvu_write64(rvu, blkaddr, 879 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 880 rvu_write64(rvu, blkaddr, 881 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 882 } 883 884 static inline u64 enable_mask(int count) 885 { 886 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 887 } 888 889 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 890 const struct npc_kpu_profile *profile) 891 { 892 int entry, num_entries, max_entries; 893 894 if (profile->cam_entries != profile->action_entries) { 895 dev_err(rvu->dev, 896 "KPU%d: CAM and action entries [%d != %d] not equal\n", 897 kpu, profile->cam_entries, profile->action_entries); 898 } 899 900 max_entries = rvu_read64(rvu, blkaddr, NPC_AF_CONST1) & 0xFFF; 901 902 /* Program CAM match entries for previous KPU extracted data */ 903 num_entries = min_t(int, profile->cam_entries, max_entries); 904 for (entry = 0; entry < num_entries; entry++) 905 npc_config_kpucam(rvu, blkaddr, 906 &profile->cam[entry], kpu, entry); 907 908 /* Program this KPU's actions */ 909 num_entries = min_t(int, profile->action_entries, max_entries); 910 for (entry = 0; entry < num_entries; entry++) 911 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 912 kpu, entry, false); 913 914 /* Enable all programmed entries */ 915 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 916 rvu_write64(rvu, blkaddr, 917 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), enable_mask(num_entries)); 918 if (num_entries > 64) { 919 rvu_write64(rvu, blkaddr, 920 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 921 enable_mask(num_entries - 64)); 922 } 923 924 /* Enable this KPU */ 925 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 926 } 927 928 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 929 { 930 profile->name = def_pfl_name; 931 profile->version = NPC_KPU_PROFILE_VER; 932 profile->ikpu = ikpu_action_entries; 933 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 934 profile->kpu = npc_kpu_profiles; 935 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 936 profile->lt_def = &npc_lt_defaults; 937 profile->mkex = &npc_mkex_default; 938 939 return 0; 940 } 941 942 static void npc_load_kpu_profile(struct rvu *rvu) 943 { 944 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 945 946 npc_prepare_default_kpu(profile); 947 } 948 949 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 950 { 951 struct rvu_hwinfo *hw = rvu->hw; 952 int num_pkinds, num_kpus, idx; 953 struct npc_pkind *pkind; 954 955 /* Get HW limits */ 956 hw->npc_kpus = (rvu_read64(rvu, blkaddr, NPC_AF_CONST) >> 8) & 0x1F; 957 958 /* Disable all KPUs and their entries */ 959 for (idx = 0; idx < hw->npc_kpus; idx++) { 960 rvu_write64(rvu, blkaddr, 961 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 962 rvu_write64(rvu, blkaddr, 963 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 964 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 965 } 966 967 /* Load and customize KPU profile. */ 968 npc_load_kpu_profile(rvu); 969 970 /* First program IKPU profile i.e PKIND configs. 971 * Check HW max count to avoid configuring junk or 972 * writing to unsupported CSR addresses. 973 */ 974 pkind = &hw->pkind; 975 num_pkinds = rvu->kpu.pkinds; 976 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 977 978 for (idx = 0; idx < num_pkinds; idx++) 979 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 980 981 /* Program KPU CAM and Action profiles */ 982 num_kpus = rvu->kpu.kpus; 983 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 984 985 for (idx = 0; idx < num_kpus; idx++) 986 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 987 } 988 989 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 990 { 991 int nixlf_count = rvu_get_nixlf_count(rvu); 992 struct npc_mcam *mcam = &rvu->hw->mcam; 993 int rsvd, err; 994 u64 cfg; 995 996 /* Get HW limits */ 997 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 998 mcam->banks = (cfg >> 44) & 0xF; 999 mcam->banksize = (cfg >> 28) & 0xFFFF; 1000 mcam->counters.max = (cfg >> 48) & 0xFFFF; 1001 1002 /* Actual number of MCAM entries vary by entry size */ 1003 cfg = (rvu_read64(rvu, blkaddr, 1004 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1005 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1006 mcam->keysize = cfg; 1007 1008 /* Number of banks combined per MCAM entry */ 1009 if (cfg == NPC_MCAM_KEY_X4) 1010 mcam->banks_per_entry = 4; 1011 else if (cfg == NPC_MCAM_KEY_X2) 1012 mcam->banks_per_entry = 2; 1013 else 1014 mcam->banks_per_entry = 1; 1015 1016 /* Reserve one MCAM entry for each of the NIX LF to 1017 * guarantee space to install default matching DMAC rule. 1018 * Also reserve 2 MCAM entries for each PF for default 1019 * channel based matching or 'bcast & promisc' matching to 1020 * support BCAST and PROMISC modes of operation for PFs. 1021 * PF0 is excluded. 1022 */ 1023 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1024 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1025 if (mcam->total_entries <= rsvd) { 1026 dev_warn(rvu->dev, 1027 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1028 mcam->total_entries); 1029 return -ENOMEM; 1030 } 1031 1032 mcam->bmap_entries = mcam->total_entries - rsvd; 1033 mcam->nixlf_offset = mcam->bmap_entries; 1034 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1035 1036 /* Allocate bitmaps for managing MCAM entries */ 1037 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1038 sizeof(long), GFP_KERNEL); 1039 if (!mcam->bmap) 1040 return -ENOMEM; 1041 1042 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1043 BITS_TO_LONGS(mcam->bmap_entries), 1044 sizeof(long), GFP_KERNEL); 1045 if (!mcam->bmap_reverse) 1046 return -ENOMEM; 1047 1048 mcam->bmap_fcnt = mcam->bmap_entries; 1049 1050 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1051 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1052 sizeof(u16), GFP_KERNEL); 1053 if (!mcam->entry2pfvf_map) 1054 return -ENOMEM; 1055 1056 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1057 * allocations and another 1/8th at the top for high priority 1058 * allocations. 1059 */ 1060 mcam->lprio_count = mcam->bmap_entries / 8; 1061 if (mcam->lprio_count > BITS_PER_LONG) 1062 mcam->lprio_count = round_down(mcam->lprio_count, 1063 BITS_PER_LONG); 1064 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1065 mcam->hprio_count = mcam->lprio_count; 1066 mcam->hprio_end = mcam->hprio_count; 1067 1068 /* Reserve last counter for MCAM RX miss action which is set to 1069 * drop pkt. This way we will know how many pkts didn't match 1070 * any MCAM entry. 1071 */ 1072 mcam->counters.max--; 1073 mcam->rx_miss_act_cntr = mcam->counters.max; 1074 1075 /* Allocate bitmap for managing MCAM counters and memory 1076 * for saving counter to RVU PFFUNC allocation mapping. 1077 */ 1078 err = rvu_alloc_bitmap(&mcam->counters); 1079 if (err) 1080 return err; 1081 1082 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1083 sizeof(u16), GFP_KERNEL); 1084 if (!mcam->cntr2pfvf_map) 1085 goto free_mem; 1086 1087 /* Alloc memory for MCAM entry to counter mapping and for tracking 1088 * counter's reference count. 1089 */ 1090 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1091 sizeof(u16), GFP_KERNEL); 1092 if (!mcam->entry2cntr_map) 1093 goto free_mem; 1094 1095 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1096 sizeof(u16), GFP_KERNEL); 1097 if (!mcam->cntr_refcnt) 1098 goto free_mem; 1099 1100 mutex_init(&mcam->lock); 1101 1102 return 0; 1103 1104 free_mem: 1105 kfree(mcam->counters.bmap); 1106 return -ENOMEM; 1107 } 1108 1109 int rvu_npc_init(struct rvu *rvu) 1110 { 1111 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1112 struct npc_pkind *pkind = &rvu->hw->pkind; 1113 struct npc_mcam *mcam = &rvu->hw->mcam; 1114 u64 cfg, nibble_ena, rx_kex, tx_kex; 1115 int blkaddr, entry, bank, err; 1116 1117 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1118 if (blkaddr < 0) { 1119 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1120 return -ENODEV; 1121 } 1122 1123 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1124 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1125 for (bank = 0; bank < ((cfg >> 44) & 0xF); bank++) { 1126 for (entry = 0; entry < ((cfg >> 28) & 0xFFFF); entry++) 1127 rvu_write64(rvu, blkaddr, 1128 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1129 } 1130 1131 /* Allocate resource bimap for pkind*/ 1132 pkind->rsrc.max = (rvu_read64(rvu, blkaddr, 1133 NPC_AF_CONST1) >> 12) & 0xFF; 1134 err = rvu_alloc_bitmap(&pkind->rsrc); 1135 if (err) 1136 return err; 1137 1138 /* Allocate mem for pkind to PF and channel mapping info */ 1139 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 1140 sizeof(u32), GFP_KERNEL); 1141 if (!pkind->pfchan_map) 1142 return -ENOMEM; 1143 1144 /* Configure KPU profile */ 1145 npc_parser_profile_init(rvu, blkaddr); 1146 1147 /* Config Outer L2, IPv4's NPC layer info */ 1148 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1149 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 1150 kpu->lt_def->pck_ol2.ltype_mask); 1151 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1152 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 1153 kpu->lt_def->pck_oip4.ltype_mask); 1154 1155 /* Config Inner IPV4 NPC layer info */ 1156 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 1157 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 1158 kpu->lt_def->pck_iip4.ltype_mask); 1159 1160 /* Enable below for Rx pkts. 1161 * - Outer IPv4 header checksum validation. 1162 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 1163 * - Inner IPv4 header checksum validation. 1164 * - Set non zero checksum error code value 1165 */ 1166 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 1167 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 1168 BIT_ULL(32) | BIT_ULL(24) | BIT_ULL(6) | 1169 BIT_ULL(2) | BIT_ULL(1)); 1170 1171 /* Set RX and TX side MCAM search key size. 1172 * LA..LD (ltype only) + Channel 1173 */ 1174 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1175 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1176 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1177 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), rx_kex); 1178 /* Due to an errata (35786) in A0 pass silicon, parse nibble enable 1179 * configuration has to be identical for both Rx and Tx interfaces. 1180 */ 1181 if (is_rvu_96xx_B0(rvu)) { 1182 tx_kex &= ~NPC_PARSE_NIBBLE; 1183 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1184 } 1185 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), tx_kex); 1186 1187 err = npc_mcam_rsrcs_init(rvu, blkaddr); 1188 if (err) 1189 return err; 1190 1191 /* Configure MKEX profile */ 1192 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 1193 1194 /* Set TX miss action to UCAST_DEFAULT i.e 1195 * transmit the packet on NIX LF SQ's default channel. 1196 */ 1197 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_TX), 1198 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1199 1200 /* If MCAM lookup doesn't result in a match, drop the received packet. 1201 * And map this action to a counter to count dropped pkts. 1202 */ 1203 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_RX), 1204 NIX_RX_ACTIONOP_DROP); 1205 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_STAT_ACT(NIX_INTF_RX), 1206 BIT_ULL(9) | mcam->rx_miss_act_cntr); 1207 1208 return 0; 1209 } 1210 1211 void rvu_npc_freemem(struct rvu *rvu) 1212 { 1213 struct npc_pkind *pkind = &rvu->hw->pkind; 1214 struct npc_mcam *mcam = &rvu->hw->mcam; 1215 1216 kfree(pkind->rsrc.bmap); 1217 kfree(mcam->counters.bmap); 1218 mutex_destroy(&mcam->lock); 1219 } 1220 1221 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 1222 int blkaddr, int *alloc_cnt, 1223 int *enable_cnt) 1224 { 1225 struct npc_mcam *mcam = &rvu->hw->mcam; 1226 int entry; 1227 1228 *alloc_cnt = 0; 1229 *enable_cnt = 0; 1230 1231 for (entry = 0; entry < mcam->bmap_entries; entry++) { 1232 if (mcam->entry2pfvf_map[entry] == pcifunc) { 1233 (*alloc_cnt)++; 1234 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 1235 (*enable_cnt)++; 1236 } 1237 } 1238 } 1239 1240 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 1241 int blkaddr, int *alloc_cnt, 1242 int *enable_cnt) 1243 { 1244 struct npc_mcam *mcam = &rvu->hw->mcam; 1245 int cntr; 1246 1247 *alloc_cnt = 0; 1248 *enable_cnt = 0; 1249 1250 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1251 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1252 (*alloc_cnt)++; 1253 if (mcam->cntr_refcnt[cntr]) 1254 (*enable_cnt)++; 1255 } 1256 } 1257 } 1258 1259 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 1260 u16 pcifunc, int entry) 1261 { 1262 /* Verify if entry is valid and if it is indeed 1263 * allocated to the requesting PFFUNC. 1264 */ 1265 if (entry >= mcam->bmap_entries) 1266 return NPC_MCAM_INVALID_REQ; 1267 1268 if (pcifunc != mcam->entry2pfvf_map[entry]) 1269 return NPC_MCAM_PERM_DENIED; 1270 1271 return 0; 1272 } 1273 1274 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 1275 u16 pcifunc, int cntr) 1276 { 1277 /* Verify if counter is valid and if it is indeed 1278 * allocated to the requesting PFFUNC. 1279 */ 1280 if (cntr >= mcam->counters.max) 1281 return NPC_MCAM_INVALID_REQ; 1282 1283 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 1284 return NPC_MCAM_PERM_DENIED; 1285 1286 return 0; 1287 } 1288 1289 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 1290 int blkaddr, u16 entry, u16 cntr) 1291 { 1292 u16 index = entry & (mcam->banksize - 1); 1293 u16 bank = npc_get_bank(mcam, entry); 1294 1295 /* Set mapping and increment counter's refcnt */ 1296 mcam->entry2cntr_map[entry] = cntr; 1297 mcam->cntr_refcnt[cntr]++; 1298 /* Enable stats */ 1299 rvu_write64(rvu, blkaddr, 1300 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 1301 BIT_ULL(9) | cntr); 1302 } 1303 1304 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 1305 struct npc_mcam *mcam, 1306 int blkaddr, u16 entry, u16 cntr) 1307 { 1308 u16 index = entry & (mcam->banksize - 1); 1309 u16 bank = npc_get_bank(mcam, entry); 1310 1311 /* Remove mapping and reduce counter's refcnt */ 1312 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 1313 mcam->cntr_refcnt[cntr]--; 1314 /* Disable stats */ 1315 rvu_write64(rvu, blkaddr, 1316 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 1317 } 1318 1319 /* Sets MCAM entry in bitmap as used. Update 1320 * reverse bitmap too. Should be called with 1321 * 'mcam->lock' held. 1322 */ 1323 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 1324 { 1325 u16 entry, rentry; 1326 1327 entry = index; 1328 rentry = mcam->bmap_entries - index - 1; 1329 1330 __set_bit(entry, mcam->bmap); 1331 __set_bit(rentry, mcam->bmap_reverse); 1332 mcam->bmap_fcnt--; 1333 } 1334 1335 /* Sets MCAM entry in bitmap as free. Update 1336 * reverse bitmap too. Should be called with 1337 * 'mcam->lock' held. 1338 */ 1339 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 1340 { 1341 u16 entry, rentry; 1342 1343 entry = index; 1344 rentry = mcam->bmap_entries - index - 1; 1345 1346 __clear_bit(entry, mcam->bmap); 1347 __clear_bit(rentry, mcam->bmap_reverse); 1348 mcam->bmap_fcnt++; 1349 } 1350 1351 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 1352 int blkaddr, u16 pcifunc) 1353 { 1354 u16 index, cntr; 1355 1356 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 1357 for (index = 0; index < mcam->bmap_entries; index++) { 1358 if (mcam->entry2pfvf_map[index] == pcifunc) { 1359 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1360 /* Free the entry in bitmap */ 1361 npc_mcam_clear_bit(mcam, index); 1362 /* Disable the entry */ 1363 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 1364 1365 /* Update entry2counter mapping */ 1366 cntr = mcam->entry2cntr_map[index]; 1367 if (cntr != NPC_MCAM_INVALID_MAP) 1368 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 1369 blkaddr, index, 1370 cntr); 1371 } 1372 } 1373 } 1374 1375 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 1376 u16 pcifunc) 1377 { 1378 u16 cntr; 1379 1380 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 1381 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1382 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1383 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1384 mcam->cntr_refcnt[cntr] = 0; 1385 rvu_free_rsrc(&mcam->counters, cntr); 1386 /* This API is expected to be called after freeing 1387 * MCAM entries, which inturn will remove 1388 * 'entry to counter' mapping. 1389 * No need to do it again. 1390 */ 1391 } 1392 } 1393 } 1394 1395 /* Find area of contiguous free entries of size 'nr'. 1396 * If not found return max contiguous free entries available. 1397 */ 1398 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 1399 u16 nr, u16 *max_area) 1400 { 1401 u16 max_area_start = 0; 1402 u16 index, next, end; 1403 1404 *max_area = 0; 1405 1406 again: 1407 index = find_next_zero_bit(map, size, start); 1408 if (index >= size) 1409 return max_area_start; 1410 1411 end = ((index + nr) >= size) ? size : index + nr; 1412 next = find_next_bit(map, end, index); 1413 if (*max_area < (next - index)) { 1414 *max_area = next - index; 1415 max_area_start = index; 1416 } 1417 1418 if (next < end) { 1419 start = next + 1; 1420 goto again; 1421 } 1422 1423 return max_area_start; 1424 } 1425 1426 /* Find number of free MCAM entries available 1427 * within range i.e in between 'start' and 'end'. 1428 */ 1429 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 1430 { 1431 u16 index, next; 1432 u16 fcnt = 0; 1433 1434 again: 1435 if (start >= end) 1436 return fcnt; 1437 1438 index = find_next_zero_bit(map, end, start); 1439 if (index >= end) 1440 return fcnt; 1441 1442 next = find_next_bit(map, end, index); 1443 if (next <= end) { 1444 fcnt += next - index; 1445 start = next + 1; 1446 goto again; 1447 } 1448 1449 fcnt += end - index; 1450 return fcnt; 1451 } 1452 1453 static void 1454 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 1455 struct npc_mcam_alloc_entry_req *req, 1456 u16 *start, u16 *end, bool *reverse) 1457 { 1458 u16 fcnt; 1459 1460 if (req->priority == NPC_MCAM_HIGHER_PRIO) 1461 goto hprio; 1462 1463 /* For a low priority entry allocation 1464 * - If reference entry is not in hprio zone then 1465 * search range: ref_entry to end. 1466 * - If reference entry is in hprio zone and if 1467 * request can be accomodated in non-hprio zone then 1468 * search range: 'start of middle zone' to 'end' 1469 * - else search in reverse, so that less number of hprio 1470 * zone entries are allocated. 1471 */ 1472 1473 *reverse = false; 1474 *start = req->ref_entry + 1; 1475 *end = mcam->bmap_entries; 1476 1477 if (req->ref_entry >= mcam->hprio_end) 1478 return; 1479 1480 fcnt = npc_mcam_get_free_count(mcam->bmap, 1481 mcam->hprio_end, mcam->bmap_entries); 1482 if (fcnt > req->count) 1483 *start = mcam->hprio_end; 1484 else 1485 *reverse = true; 1486 return; 1487 1488 hprio: 1489 /* For a high priority entry allocation, search is always 1490 * in reverse to preserve hprio zone entries. 1491 * - If reference entry is not in lprio zone then 1492 * search range: 0 to ref_entry. 1493 * - If reference entry is in lprio zone and if 1494 * request can be accomodated in middle zone then 1495 * search range: 'hprio_end' to 'lprio_start' 1496 */ 1497 1498 *reverse = true; 1499 *start = 0; 1500 *end = req->ref_entry; 1501 1502 if (req->ref_entry <= mcam->lprio_start) 1503 return; 1504 1505 fcnt = npc_mcam_get_free_count(mcam->bmap, 1506 mcam->hprio_end, mcam->lprio_start); 1507 if (fcnt < req->count) 1508 return; 1509 *start = mcam->hprio_end; 1510 *end = mcam->lprio_start; 1511 } 1512 1513 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 1514 struct npc_mcam_alloc_entry_req *req, 1515 struct npc_mcam_alloc_entry_rsp *rsp) 1516 { 1517 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 1518 u16 fcnt, hp_fcnt, lp_fcnt; 1519 u16 start, end, index; 1520 int entry, next_start; 1521 bool reverse = false; 1522 unsigned long *bmap; 1523 u16 max_contig; 1524 1525 mutex_lock(&mcam->lock); 1526 1527 /* Check if there are any free entries */ 1528 if (!mcam->bmap_fcnt) { 1529 mutex_unlock(&mcam->lock); 1530 return NPC_MCAM_ALLOC_FAILED; 1531 } 1532 1533 /* MCAM entries are divided into high priority, middle and 1534 * low priority zones. Idea is to not allocate top and lower 1535 * most entries as much as possible, this is to increase 1536 * probability of honouring priority allocation requests. 1537 * 1538 * Two bitmaps are used for mcam entry management, 1539 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 1540 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 1541 * 1542 * Reverse bitmap is used to allocate entries 1543 * - when a higher priority entry is requested 1544 * - when available free entries are less. 1545 * Lower priority ones out of avaialble free entries are always 1546 * chosen when 'high vs low' question arises. 1547 */ 1548 1549 /* Get the search range for priority allocation request */ 1550 if (req->priority) { 1551 npc_get_mcam_search_range_priority(mcam, req, 1552 &start, &end, &reverse); 1553 goto alloc; 1554 } 1555 1556 /* Find out the search range for non-priority allocation request 1557 * 1558 * Get MCAM free entry count in middle zone. 1559 */ 1560 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 1561 mcam->lprio_start, 1562 mcam->bmap_entries); 1563 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 1564 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 1565 1566 /* Check if request can be accomodated in the middle zone */ 1567 if (fcnt > req->count) { 1568 start = mcam->hprio_end; 1569 end = mcam->lprio_start; 1570 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 1571 /* Expand search zone from half of hprio zone to 1572 * half of lprio zone. 1573 */ 1574 start = mcam->hprio_end / 2; 1575 end = mcam->bmap_entries - (mcam->lprio_count / 2); 1576 reverse = true; 1577 } else { 1578 /* Not enough free entries, search all entries in reverse, 1579 * so that low priority ones will get used up. 1580 */ 1581 reverse = true; 1582 start = 0; 1583 end = mcam->bmap_entries; 1584 } 1585 1586 alloc: 1587 if (reverse) { 1588 bmap = mcam->bmap_reverse; 1589 start = mcam->bmap_entries - start; 1590 end = mcam->bmap_entries - end; 1591 index = start; 1592 start = end; 1593 end = index; 1594 } else { 1595 bmap = mcam->bmap; 1596 } 1597 1598 if (req->contig) { 1599 /* Allocate requested number of contiguous entries, if 1600 * unsuccessful find max contiguous entries available. 1601 */ 1602 index = npc_mcam_find_zero_area(bmap, end, start, 1603 req->count, &max_contig); 1604 rsp->count = max_contig; 1605 if (reverse) 1606 rsp->entry = mcam->bmap_entries - index - max_contig; 1607 else 1608 rsp->entry = index; 1609 } else { 1610 /* Allocate requested number of non-contiguous entries, 1611 * if unsuccessful allocate as many as possible. 1612 */ 1613 rsp->count = 0; 1614 next_start = start; 1615 for (entry = 0; entry < req->count; entry++) { 1616 index = find_next_zero_bit(bmap, end, next_start); 1617 if (index >= end) 1618 break; 1619 1620 next_start = start + (index - start) + 1; 1621 1622 /* Save the entry's index */ 1623 if (reverse) 1624 index = mcam->bmap_entries - index - 1; 1625 entry_list[entry] = index; 1626 rsp->count++; 1627 } 1628 } 1629 1630 /* If allocating requested no of entries is unsucessful, 1631 * expand the search range to full bitmap length and retry. 1632 */ 1633 if (!req->priority && (rsp->count < req->count) && 1634 ((end - start) != mcam->bmap_entries)) { 1635 reverse = true; 1636 start = 0; 1637 end = mcam->bmap_entries; 1638 goto alloc; 1639 } 1640 1641 /* For priority entry allocation requests, if allocation is 1642 * failed then expand search to max possible range and retry. 1643 */ 1644 if (req->priority && rsp->count < req->count) { 1645 if (req->priority == NPC_MCAM_LOWER_PRIO && 1646 (start != (req->ref_entry + 1))) { 1647 start = req->ref_entry + 1; 1648 end = mcam->bmap_entries; 1649 reverse = false; 1650 goto alloc; 1651 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 1652 ((end - start) != req->ref_entry)) { 1653 start = 0; 1654 end = req->ref_entry; 1655 reverse = true; 1656 goto alloc; 1657 } 1658 } 1659 1660 /* Copy MCAM entry indices into mbox response entry_list. 1661 * Requester always expects indices in ascending order, so 1662 * so reverse the list if reverse bitmap is used for allocation. 1663 */ 1664 if (!req->contig && rsp->count) { 1665 index = 0; 1666 for (entry = rsp->count - 1; entry >= 0; entry--) { 1667 if (reverse) 1668 rsp->entry_list[index++] = entry_list[entry]; 1669 else 1670 rsp->entry_list[entry] = entry_list[entry]; 1671 } 1672 } 1673 1674 /* Mark the allocated entries as used and set nixlf mapping */ 1675 for (entry = 0; entry < rsp->count; entry++) { 1676 index = req->contig ? 1677 (rsp->entry + entry) : rsp->entry_list[entry]; 1678 npc_mcam_set_bit(mcam, index); 1679 mcam->entry2pfvf_map[index] = pcifunc; 1680 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1681 } 1682 1683 /* Update available free count in mbox response */ 1684 rsp->free_count = mcam->bmap_fcnt; 1685 1686 mutex_unlock(&mcam->lock); 1687 return 0; 1688 } 1689 1690 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 1691 struct npc_mcam_alloc_entry_req *req, 1692 struct npc_mcam_alloc_entry_rsp *rsp) 1693 { 1694 struct npc_mcam *mcam = &rvu->hw->mcam; 1695 u16 pcifunc = req->hdr.pcifunc; 1696 int blkaddr; 1697 1698 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1699 if (blkaddr < 0) 1700 return NPC_MCAM_INVALID_REQ; 1701 1702 rsp->entry = NPC_MCAM_ENTRY_INVALID; 1703 rsp->free_count = 0; 1704 1705 /* Check if ref_entry is within range */ 1706 if (req->priority && req->ref_entry >= mcam->bmap_entries) 1707 return NPC_MCAM_INVALID_REQ; 1708 1709 /* ref_entry can't be '0' if requested priority is high. 1710 * Can't be last entry if requested priority is low. 1711 */ 1712 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 1713 ((req->ref_entry == (mcam->bmap_entries - 1)) && 1714 req->priority == NPC_MCAM_LOWER_PRIO)) 1715 return NPC_MCAM_INVALID_REQ; 1716 1717 /* Since list of allocated indices needs to be sent to requester, 1718 * max number of non-contiguous entries per mbox msg is limited. 1719 */ 1720 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) 1721 return NPC_MCAM_INVALID_REQ; 1722 1723 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 1724 if (!is_nixlf_attached(rvu, pcifunc)) 1725 return NPC_MCAM_ALLOC_DENIED; 1726 1727 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 1728 } 1729 1730 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 1731 struct npc_mcam_free_entry_req *req, 1732 struct msg_rsp *rsp) 1733 { 1734 struct npc_mcam *mcam = &rvu->hw->mcam; 1735 u16 pcifunc = req->hdr.pcifunc; 1736 int blkaddr, rc = 0; 1737 u16 cntr; 1738 1739 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1740 if (blkaddr < 0) 1741 return NPC_MCAM_INVALID_REQ; 1742 1743 /* Free request from PFFUNC with no NIXLF attached, ignore */ 1744 if (!is_nixlf_attached(rvu, pcifunc)) 1745 return NPC_MCAM_INVALID_REQ; 1746 1747 mutex_lock(&mcam->lock); 1748 1749 if (req->all) 1750 goto free_all; 1751 1752 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1753 if (rc) 1754 goto exit; 1755 1756 mcam->entry2pfvf_map[req->entry] = 0; 1757 npc_mcam_clear_bit(mcam, req->entry); 1758 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1759 1760 /* Update entry2counter mapping */ 1761 cntr = mcam->entry2cntr_map[req->entry]; 1762 if (cntr != NPC_MCAM_INVALID_MAP) 1763 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1764 req->entry, cntr); 1765 1766 goto exit; 1767 1768 free_all: 1769 /* Free up all entries allocated to requesting PFFUNC */ 1770 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1771 exit: 1772 mutex_unlock(&mcam->lock); 1773 return rc; 1774 } 1775 1776 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 1777 struct npc_mcam_write_entry_req *req, 1778 struct msg_rsp *rsp) 1779 { 1780 struct npc_mcam *mcam = &rvu->hw->mcam; 1781 u16 pcifunc = req->hdr.pcifunc; 1782 int blkaddr, rc; 1783 1784 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1785 if (blkaddr < 0) 1786 return NPC_MCAM_INVALID_REQ; 1787 1788 mutex_lock(&mcam->lock); 1789 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1790 if (rc) 1791 goto exit; 1792 1793 if (req->set_cntr && 1794 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 1795 rc = NPC_MCAM_INVALID_REQ; 1796 goto exit; 1797 } 1798 1799 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) { 1800 rc = NPC_MCAM_INVALID_REQ; 1801 goto exit; 1802 } 1803 1804 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, req->intf, 1805 &req->entry_data, req->enable_entry); 1806 1807 if (req->set_cntr) 1808 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1809 req->entry, req->cntr); 1810 1811 rc = 0; 1812 exit: 1813 mutex_unlock(&mcam->lock); 1814 return rc; 1815 } 1816 1817 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 1818 struct npc_mcam_ena_dis_entry_req *req, 1819 struct msg_rsp *rsp) 1820 { 1821 struct npc_mcam *mcam = &rvu->hw->mcam; 1822 u16 pcifunc = req->hdr.pcifunc; 1823 int blkaddr, rc; 1824 1825 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1826 if (blkaddr < 0) 1827 return NPC_MCAM_INVALID_REQ; 1828 1829 mutex_lock(&mcam->lock); 1830 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1831 mutex_unlock(&mcam->lock); 1832 if (rc) 1833 return rc; 1834 1835 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 1836 1837 return 0; 1838 } 1839 1840 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 1841 struct npc_mcam_ena_dis_entry_req *req, 1842 struct msg_rsp *rsp) 1843 { 1844 struct npc_mcam *mcam = &rvu->hw->mcam; 1845 u16 pcifunc = req->hdr.pcifunc; 1846 int blkaddr, rc; 1847 1848 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1849 if (blkaddr < 0) 1850 return NPC_MCAM_INVALID_REQ; 1851 1852 mutex_lock(&mcam->lock); 1853 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1854 mutex_unlock(&mcam->lock); 1855 if (rc) 1856 return rc; 1857 1858 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1859 1860 return 0; 1861 } 1862 1863 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 1864 struct npc_mcam_shift_entry_req *req, 1865 struct npc_mcam_shift_entry_rsp *rsp) 1866 { 1867 struct npc_mcam *mcam = &rvu->hw->mcam; 1868 u16 pcifunc = req->hdr.pcifunc; 1869 u16 old_entry, new_entry; 1870 u16 index, cntr; 1871 int blkaddr, rc; 1872 1873 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1874 if (blkaddr < 0) 1875 return NPC_MCAM_INVALID_REQ; 1876 1877 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 1878 return NPC_MCAM_INVALID_REQ; 1879 1880 mutex_lock(&mcam->lock); 1881 for (index = 0; index < req->shift_count; index++) { 1882 old_entry = req->curr_entry[index]; 1883 new_entry = req->new_entry[index]; 1884 1885 /* Check if both old and new entries are valid and 1886 * does belong to this PFFUNC or not. 1887 */ 1888 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 1889 if (rc) 1890 break; 1891 1892 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 1893 if (rc) 1894 break; 1895 1896 /* new_entry should not have a counter mapped */ 1897 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 1898 rc = NPC_MCAM_PERM_DENIED; 1899 break; 1900 } 1901 1902 /* Disable the new_entry */ 1903 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 1904 1905 /* Copy rule from old entry to new entry */ 1906 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 1907 1908 /* Copy counter mapping, if any */ 1909 cntr = mcam->entry2cntr_map[old_entry]; 1910 if (cntr != NPC_MCAM_INVALID_MAP) { 1911 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1912 old_entry, cntr); 1913 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1914 new_entry, cntr); 1915 } 1916 1917 /* Enable new_entry and disable old_entry */ 1918 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 1919 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 1920 } 1921 1922 /* If shift has failed then report the failed index */ 1923 if (index != req->shift_count) { 1924 rc = NPC_MCAM_PERM_DENIED; 1925 rsp->failed_entry_idx = index; 1926 } 1927 1928 mutex_unlock(&mcam->lock); 1929 return rc; 1930 } 1931 1932 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 1933 struct npc_mcam_alloc_counter_req *req, 1934 struct npc_mcam_alloc_counter_rsp *rsp) 1935 { 1936 struct npc_mcam *mcam = &rvu->hw->mcam; 1937 u16 pcifunc = req->hdr.pcifunc; 1938 u16 max_contig, cntr; 1939 int blkaddr, index; 1940 1941 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1942 if (blkaddr < 0) 1943 return NPC_MCAM_INVALID_REQ; 1944 1945 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 1946 if (!is_nixlf_attached(rvu, pcifunc)) 1947 return NPC_MCAM_INVALID_REQ; 1948 1949 /* Since list of allocated counter IDs needs to be sent to requester, 1950 * max number of non-contiguous counters per mbox msg is limited. 1951 */ 1952 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 1953 return NPC_MCAM_INVALID_REQ; 1954 1955 mutex_lock(&mcam->lock); 1956 1957 /* Check if unused counters are available or not */ 1958 if (!rvu_rsrc_free_count(&mcam->counters)) { 1959 mutex_unlock(&mcam->lock); 1960 return NPC_MCAM_ALLOC_FAILED; 1961 } 1962 1963 rsp->count = 0; 1964 1965 if (req->contig) { 1966 /* Allocate requested number of contiguous counters, if 1967 * unsuccessful find max contiguous entries available. 1968 */ 1969 index = npc_mcam_find_zero_area(mcam->counters.bmap, 1970 mcam->counters.max, 0, 1971 req->count, &max_contig); 1972 rsp->count = max_contig; 1973 rsp->cntr = index; 1974 for (cntr = index; cntr < (index + max_contig); cntr++) { 1975 __set_bit(cntr, mcam->counters.bmap); 1976 mcam->cntr2pfvf_map[cntr] = pcifunc; 1977 } 1978 } else { 1979 /* Allocate requested number of non-contiguous counters, 1980 * if unsuccessful allocate as many as possible. 1981 */ 1982 for (cntr = 0; cntr < req->count; cntr++) { 1983 index = rvu_alloc_rsrc(&mcam->counters); 1984 if (index < 0) 1985 break; 1986 rsp->cntr_list[cntr] = index; 1987 rsp->count++; 1988 mcam->cntr2pfvf_map[index] = pcifunc; 1989 } 1990 } 1991 1992 mutex_unlock(&mcam->lock); 1993 return 0; 1994 } 1995 1996 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 1997 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 1998 { 1999 struct npc_mcam *mcam = &rvu->hw->mcam; 2000 u16 index, entry = 0; 2001 int blkaddr, err; 2002 2003 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2004 if (blkaddr < 0) 2005 return NPC_MCAM_INVALID_REQ; 2006 2007 mutex_lock(&mcam->lock); 2008 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2009 if (err) { 2010 mutex_unlock(&mcam->lock); 2011 return err; 2012 } 2013 2014 /* Mark counter as free/unused */ 2015 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2016 rvu_free_rsrc(&mcam->counters, req->cntr); 2017 2018 /* Disable all MCAM entry's stats which are using this counter */ 2019 while (entry < mcam->bmap_entries) { 2020 if (!mcam->cntr_refcnt[req->cntr]) 2021 break; 2022 2023 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2024 if (index >= mcam->bmap_entries) 2025 break; 2026 if (mcam->entry2cntr_map[index] != req->cntr) 2027 continue; 2028 2029 entry = index + 1; 2030 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2031 index, req->cntr); 2032 } 2033 2034 mutex_unlock(&mcam->lock); 2035 return 0; 2036 } 2037 2038 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2039 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2040 { 2041 struct npc_mcam *mcam = &rvu->hw->mcam; 2042 u16 index, entry = 0; 2043 int blkaddr, rc; 2044 2045 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2046 if (blkaddr < 0) 2047 return NPC_MCAM_INVALID_REQ; 2048 2049 mutex_lock(&mcam->lock); 2050 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2051 if (rc) 2052 goto exit; 2053 2054 /* Unmap the MCAM entry and counter */ 2055 if (!req->all) { 2056 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2057 if (rc) 2058 goto exit; 2059 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2060 req->entry, req->cntr); 2061 goto exit; 2062 } 2063 2064 /* Disable all MCAM entry's stats which are using this counter */ 2065 while (entry < mcam->bmap_entries) { 2066 if (!mcam->cntr_refcnt[req->cntr]) 2067 break; 2068 2069 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2070 if (index >= mcam->bmap_entries) 2071 break; 2072 if (mcam->entry2cntr_map[index] != req->cntr) 2073 continue; 2074 2075 entry = index + 1; 2076 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2077 index, req->cntr); 2078 } 2079 exit: 2080 mutex_unlock(&mcam->lock); 2081 return rc; 2082 } 2083 2084 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2085 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2086 { 2087 struct npc_mcam *mcam = &rvu->hw->mcam; 2088 int blkaddr, err; 2089 2090 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2091 if (blkaddr < 0) 2092 return NPC_MCAM_INVALID_REQ; 2093 2094 mutex_lock(&mcam->lock); 2095 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2096 mutex_unlock(&mcam->lock); 2097 if (err) 2098 return err; 2099 2100 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 2101 2102 return 0; 2103 } 2104 2105 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 2106 struct npc_mcam_oper_counter_req *req, 2107 struct npc_mcam_oper_counter_rsp *rsp) 2108 { 2109 struct npc_mcam *mcam = &rvu->hw->mcam; 2110 int blkaddr, err; 2111 2112 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2113 if (blkaddr < 0) 2114 return NPC_MCAM_INVALID_REQ; 2115 2116 mutex_lock(&mcam->lock); 2117 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2118 mutex_unlock(&mcam->lock); 2119 if (err) 2120 return err; 2121 2122 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 2123 rsp->stat &= BIT_ULL(48) - 1; 2124 2125 return 0; 2126 } 2127 2128 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 2129 struct npc_mcam_alloc_and_write_entry_req *req, 2130 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 2131 { 2132 struct npc_mcam_alloc_counter_req cntr_req; 2133 struct npc_mcam_alloc_counter_rsp cntr_rsp; 2134 struct npc_mcam_alloc_entry_req entry_req; 2135 struct npc_mcam_alloc_entry_rsp entry_rsp; 2136 struct npc_mcam *mcam = &rvu->hw->mcam; 2137 u16 entry = NPC_MCAM_ENTRY_INVALID; 2138 u16 cntr = NPC_MCAM_ENTRY_INVALID; 2139 int blkaddr, rc; 2140 2141 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2142 if (blkaddr < 0) 2143 return NPC_MCAM_INVALID_REQ; 2144 2145 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) 2146 return NPC_MCAM_INVALID_REQ; 2147 2148 /* Try to allocate a MCAM entry */ 2149 entry_req.hdr.pcifunc = req->hdr.pcifunc; 2150 entry_req.contig = true; 2151 entry_req.priority = req->priority; 2152 entry_req.ref_entry = req->ref_entry; 2153 entry_req.count = 1; 2154 2155 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 2156 &entry_req, &entry_rsp); 2157 if (rc) 2158 return rc; 2159 2160 if (!entry_rsp.count) 2161 return NPC_MCAM_ALLOC_FAILED; 2162 2163 entry = entry_rsp.entry; 2164 2165 if (!req->alloc_cntr) 2166 goto write_entry; 2167 2168 /* Now allocate counter */ 2169 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 2170 cntr_req.contig = true; 2171 cntr_req.count = 1; 2172 2173 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 2174 if (rc) { 2175 /* Free allocated MCAM entry */ 2176 mutex_lock(&mcam->lock); 2177 mcam->entry2pfvf_map[entry] = 0; 2178 npc_mcam_clear_bit(mcam, entry); 2179 mutex_unlock(&mcam->lock); 2180 return rc; 2181 } 2182 2183 cntr = cntr_rsp.cntr; 2184 2185 write_entry: 2186 mutex_lock(&mcam->lock); 2187 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, req->intf, 2188 &req->entry_data, req->enable_entry); 2189 2190 if (req->alloc_cntr) 2191 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 2192 mutex_unlock(&mcam->lock); 2193 2194 rsp->entry = entry; 2195 rsp->cntr = cntr; 2196 2197 return 0; 2198 } 2199 2200 #define GET_KEX_CFG(intf) \ 2201 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 2202 2203 #define GET_KEX_FLAGS(ld) \ 2204 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 2205 2206 #define GET_KEX_LD(intf, lid, lt, ld) \ 2207 rvu_read64(rvu, BLKADDR_NPC, \ 2208 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 2209 2210 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 2211 rvu_read64(rvu, BLKADDR_NPC, \ 2212 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 2213 2214 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 2215 struct npc_get_kex_cfg_rsp *rsp) 2216 { 2217 int lid, lt, ld, fl; 2218 2219 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 2220 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 2221 for (lid = 0; lid < NPC_MAX_LID; lid++) { 2222 for (lt = 0; lt < NPC_MAX_LT; lt++) { 2223 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2224 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 2225 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 2226 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 2227 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 2228 } 2229 } 2230 } 2231 for (ld = 0; ld < NPC_MAX_LD; ld++) 2232 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 2233 2234 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2235 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 2236 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 2237 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 2238 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 2239 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 2240 } 2241 } 2242 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 2243 return 0; 2244 } 2245 2246 int rvu_npc_update_rxvlan(struct rvu *rvu, u16 pcifunc, int nixlf) 2247 { 2248 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 2249 struct npc_mcam *mcam = &rvu->hw->mcam; 2250 int blkaddr, index; 2251 bool enable; 2252 2253 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2254 if (blkaddr < 0) 2255 return NIX_AF_ERR_AF_LF_INVALID; 2256 2257 if (!pfvf->rxvlan) 2258 return 0; 2259 2260 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 2261 NIXLF_UCAST_ENTRY); 2262 pfvf->entry.action = npc_get_mcam_action(rvu, mcam, blkaddr, index); 2263 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, index); 2264 npc_config_mcam_entry(rvu, mcam, blkaddr, pfvf->rxvlan_index, 2265 NIX_INTF_RX, &pfvf->entry, enable); 2266 2267 return 0; 2268 } 2269