1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "cgx.h" 20 #include "npc_profile.h" 21 22 #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ 23 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 24 25 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 26 #define NPC_HW_TSTAMP_OFFSET 8 27 #define NPC_KEX_CHAN_MASK 0xFFFULL 28 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 29 30 #define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8)) 31 32 static const char def_pfl_name[] = "default"; 33 34 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 35 int blkaddr, u16 pcifunc); 36 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 37 u16 pcifunc); 38 39 bool is_npc_intf_tx(u8 intf) 40 { 41 return !!(intf & 0x1); 42 } 43 44 bool is_npc_intf_rx(u8 intf) 45 { 46 return !(intf & 0x1); 47 } 48 49 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 50 { 51 struct rvu_hwinfo *hw = rvu->hw; 52 53 return intf < hw->npc_intfs; 54 } 55 56 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 57 { 58 /* Due to a HW issue in these silicon versions, parse nibble enable 59 * configuration has to be identical for both Rx and Tx interfaces. 60 */ 61 if (is_rvu_96xx_B0(rvu)) 62 return nibble_ena; 63 return 0; 64 } 65 66 static int npc_mcam_verify_pf_func(struct rvu *rvu, 67 struct mcam_entry *entry_data, u8 intf, 68 u16 pcifunc) 69 { 70 u16 pf_func, pf_func_mask; 71 72 if (is_npc_intf_rx(intf)) 73 return 0; 74 75 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 76 NPC_KEX_PF_FUNC_MASK; 77 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 78 79 pf_func = be16_to_cpu((__force __be16)pf_func); 80 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 81 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 82 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 83 return -EINVAL; 84 85 return 0; 86 } 87 88 int npc_mcam_verify_channel(struct rvu *rvu, u16 pcifunc, u8 intf, u16 channel) 89 { 90 int pf = rvu_get_pf(pcifunc); 91 u8 cgx_id, lmac_id; 92 int base = 0, end; 93 94 if (is_npc_intf_tx(intf)) 95 return 0; 96 97 /* return in case of AF installed rules */ 98 if (is_pffunc_af(pcifunc)) 99 return 0; 100 101 if (is_afvf(pcifunc)) { 102 end = rvu_get_num_lbk_chans(); 103 if (end < 0) 104 return -EINVAL; 105 } else { 106 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 107 base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0x0); 108 /* CGX mapped functions has maximum of 16 channels */ 109 end = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0xF); 110 } 111 112 if (channel < base || channel > end) 113 return -EINVAL; 114 115 return 0; 116 } 117 118 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 119 { 120 int blkaddr; 121 u64 val = 0; 122 123 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 124 if (blkaddr < 0) 125 return; 126 127 /* Config CPI base for the PKIND */ 128 val = pkind | 1ULL << 62; 129 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 130 } 131 132 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 133 { 134 struct npc_pkind *pkind = &rvu->hw->pkind; 135 u32 map; 136 int i; 137 138 for (i = 0; i < pkind->rsrc.max; i++) { 139 map = pkind->pfchan_map[i]; 140 if (((map >> 16) & 0x3F) == pf) 141 return i; 142 } 143 return -1; 144 } 145 146 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 147 148 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 149 { 150 int pkind, blkaddr; 151 u64 val; 152 153 pkind = rvu_npc_get_pkind(rvu, pf); 154 if (pkind < 0) { 155 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 156 return -EINVAL; 157 } 158 159 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 160 if (blkaddr < 0) { 161 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 162 return -EINVAL; 163 } 164 165 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 166 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 167 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 168 if (enable) 169 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 170 NPC_HW_TSTAMP_OFFSET); 171 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 172 173 return 0; 174 } 175 176 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 177 int nixlf) 178 { 179 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 180 struct rvu *rvu = hw->rvu; 181 int blkaddr = 0, max = 0; 182 struct rvu_block *block; 183 struct rvu_pfvf *pfvf; 184 185 pfvf = rvu_get_pfvf(rvu, pcifunc); 186 /* Given a PF/VF and NIX LF number calculate the unicast mcam 187 * entry index based on the NIX block assigned to the PF/VF. 188 */ 189 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 190 while (blkaddr) { 191 if (pfvf->nix_blkaddr == blkaddr) 192 break; 193 block = &rvu->hw->block[blkaddr]; 194 max += block->lf.max; 195 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 196 } 197 198 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 199 } 200 201 int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 202 u16 pcifunc, int nixlf, int type) 203 { 204 int pf = rvu_get_pf(pcifunc); 205 int index; 206 207 /* Check if this is for a PF */ 208 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 209 /* Reserved entries exclude PF0 */ 210 pf--; 211 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 212 /* Broadcast address matching entry should be first so 213 * that the packet can be replicated to all VFs. 214 */ 215 if (type == NIXLF_BCAST_ENTRY) 216 return index; 217 else if (type == NIXLF_ALLMULTI_ENTRY) 218 return index + 1; 219 else if (type == NIXLF_PROMISC_ENTRY) 220 return index + 2; 221 } 222 223 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 224 } 225 226 int npc_get_bank(struct npc_mcam *mcam, int index) 227 { 228 int bank = index / mcam->banksize; 229 230 /* 0,1 & 2,3 banks are combined for this keysize */ 231 if (mcam->keysize == NPC_MCAM_KEY_X2) 232 return bank ? 2 : 0; 233 234 return bank; 235 } 236 237 bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 238 int blkaddr, int index) 239 { 240 int bank = npc_get_bank(mcam, index); 241 u64 cfg; 242 243 index &= (mcam->banksize - 1); 244 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 245 return (cfg & 1); 246 } 247 248 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 249 int blkaddr, int index, bool enable) 250 { 251 int bank = npc_get_bank(mcam, index); 252 int actbank = bank; 253 254 index &= (mcam->banksize - 1); 255 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 256 rvu_write64(rvu, blkaddr, 257 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 258 enable ? 1 : 0); 259 } 260 } 261 262 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 263 int blkaddr, int index) 264 { 265 int bank = npc_get_bank(mcam, index); 266 int actbank = bank; 267 268 index &= (mcam->banksize - 1); 269 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 270 rvu_write64(rvu, blkaddr, 271 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 272 rvu_write64(rvu, blkaddr, 273 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 274 275 rvu_write64(rvu, blkaddr, 276 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 277 rvu_write64(rvu, blkaddr, 278 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 279 280 rvu_write64(rvu, blkaddr, 281 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 282 rvu_write64(rvu, blkaddr, 283 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 284 } 285 } 286 287 static void npc_get_keyword(struct mcam_entry *entry, int idx, 288 u64 *cam0, u64 *cam1) 289 { 290 u64 kw_mask = 0x00; 291 292 #define CAM_MASK(n) (BIT_ULL(n) - 1) 293 294 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 295 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 296 * 297 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 298 */ 299 switch (idx) { 300 case 0: 301 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 302 *cam1 = entry->kw[0]; 303 kw_mask = entry->kw_mask[0]; 304 break; 305 case 1: 306 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 307 *cam1 = entry->kw[1] & CAM_MASK(48); 308 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 309 break; 310 case 2: 311 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 312 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 313 */ 314 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 315 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 316 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 317 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 318 break; 319 case 3: 320 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 321 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 322 */ 323 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 324 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 325 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 326 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 327 break; 328 case 4: 329 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 330 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 331 */ 332 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 333 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 334 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 335 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 336 break; 337 case 5: 338 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 339 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 340 */ 341 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 342 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 343 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 344 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 345 break; 346 case 6: 347 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 348 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 349 */ 350 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 351 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 352 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 353 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 354 break; 355 case 7: 356 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 357 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 358 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 359 break; 360 } 361 362 *cam1 &= kw_mask; 363 *cam0 = ~*cam1 & kw_mask; 364 } 365 366 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 367 u64 cam0, u64 cam1) 368 { 369 /* Similar to npc_get_keyword, but fills mcam_entry structure from 370 * CAM registers. 371 */ 372 switch (idx) { 373 case 0: 374 entry->kw[0] = cam1; 375 entry->kw_mask[0] = cam1 ^ cam0; 376 break; 377 case 1: 378 entry->kw[1] = cam1; 379 entry->kw_mask[1] = cam1 ^ cam0; 380 break; 381 case 2: 382 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 383 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 384 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 385 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 386 break; 387 case 3: 388 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 389 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 390 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 391 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 392 break; 393 case 4: 394 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 395 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 396 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 397 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 398 break; 399 case 5: 400 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 401 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 402 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 403 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 404 break; 405 case 6: 406 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 407 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 408 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 409 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 410 break; 411 case 7: 412 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 413 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 414 break; 415 } 416 } 417 418 static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 419 int blkaddr, u16 pf_func) 420 { 421 int bank, nixlf, index; 422 423 /* get ucast entry rule entry index */ 424 nix_get_nixlf(rvu, pf_func, &nixlf, NULL); 425 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf, 426 NIXLF_UCAST_ENTRY); 427 bank = npc_get_bank(mcam, index); 428 index &= (mcam->banksize - 1); 429 430 return rvu_read64(rvu, blkaddr, 431 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 432 } 433 434 static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, 435 int blkaddr, int index, struct mcam_entry *entry, 436 bool *enable) 437 { 438 u16 owner, target_func; 439 struct rvu_pfvf *pfvf; 440 u64 rx_action; 441 442 owner = mcam->entry2pfvf_map[index]; 443 target_func = (entry->action >> 4) & 0xffff; 444 /* do nothing when target is LBK/PF or owner is not PF */ 445 if (is_afvf(target_func) || (owner & RVU_PFVF_FUNC_MASK) || 446 !(target_func & RVU_PFVF_FUNC_MASK)) 447 return; 448 449 /* save entry2target_pffunc */ 450 pfvf = rvu_get_pfvf(rvu, target_func); 451 mcam->entry2target_pffunc[index] = target_func; 452 453 /* don't enable rule when nixlf not attached or initialized */ 454 if (!(is_nixlf_attached(rvu, target_func) && 455 test_bit(NIXLF_INITIALIZED, &pfvf->flags))) 456 *enable = false; 457 458 /* copy VF default entry action to the VF mcam entry */ 459 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr, 460 target_func); 461 if (rx_action) 462 entry->action = rx_action; 463 } 464 465 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 466 int blkaddr, int index, u8 intf, 467 struct mcam_entry *entry, bool enable) 468 { 469 int bank = npc_get_bank(mcam, index); 470 int kw = 0, actbank, actindex; 471 u64 cam0, cam1; 472 473 actbank = bank; /* Save bank id, to set action later on */ 474 actindex = index; 475 index &= (mcam->banksize - 1); 476 477 /* Disable before mcam entry update */ 478 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 479 480 /* Clear mcam entry to avoid writes being suppressed by NPC */ 481 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 482 483 /* CAM1 takes the comparison value and 484 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 485 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 486 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 487 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 488 */ 489 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 490 /* Interface should be set in all banks */ 491 rvu_write64(rvu, blkaddr, 492 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 493 intf); 494 rvu_write64(rvu, blkaddr, 495 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 496 ~intf & 0x3); 497 498 /* Set the match key */ 499 npc_get_keyword(entry, kw, &cam0, &cam1); 500 rvu_write64(rvu, blkaddr, 501 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 502 rvu_write64(rvu, blkaddr, 503 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 504 505 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 506 rvu_write64(rvu, blkaddr, 507 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 508 rvu_write64(rvu, blkaddr, 509 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 510 } 511 512 /* PF installing VF rule */ 513 if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) 514 npc_fixup_vf_rule(rvu, mcam, blkaddr, index, entry, &enable); 515 516 /* Set 'action' */ 517 rvu_write64(rvu, blkaddr, 518 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 519 520 /* Set TAG 'action' */ 521 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 522 entry->vtag_action); 523 524 /* Enable the entry */ 525 if (enable) 526 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 527 } 528 529 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 530 int blkaddr, u16 src, 531 struct mcam_entry *entry, u8 *intf, u8 *ena) 532 { 533 int sbank = npc_get_bank(mcam, src); 534 int bank, kw = 0; 535 u64 cam0, cam1; 536 537 src &= (mcam->banksize - 1); 538 bank = sbank; 539 540 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 541 cam1 = rvu_read64(rvu, blkaddr, 542 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 543 cam0 = rvu_read64(rvu, blkaddr, 544 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 545 npc_fill_entryword(entry, kw, cam0, cam1); 546 547 cam1 = rvu_read64(rvu, blkaddr, 548 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 549 cam0 = rvu_read64(rvu, blkaddr, 550 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 551 npc_fill_entryword(entry, kw + 1, cam0, cam1); 552 } 553 554 entry->action = rvu_read64(rvu, blkaddr, 555 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 556 entry->vtag_action = 557 rvu_read64(rvu, blkaddr, 558 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 559 *intf = rvu_read64(rvu, blkaddr, 560 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 561 *ena = rvu_read64(rvu, blkaddr, 562 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 563 } 564 565 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 566 int blkaddr, u16 src, u16 dest) 567 { 568 int dbank = npc_get_bank(mcam, dest); 569 int sbank = npc_get_bank(mcam, src); 570 u64 cfg, sreg, dreg; 571 int bank, i; 572 573 src &= (mcam->banksize - 1); 574 dest &= (mcam->banksize - 1); 575 576 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 577 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 578 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 579 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 580 for (i = 0; i < 6; i++) { 581 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 582 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 583 } 584 } 585 586 /* Copy action */ 587 cfg = rvu_read64(rvu, blkaddr, 588 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 589 rvu_write64(rvu, blkaddr, 590 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 591 592 /* Copy TAG action */ 593 cfg = rvu_read64(rvu, blkaddr, 594 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 595 rvu_write64(rvu, blkaddr, 596 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 597 598 /* Enable or disable */ 599 cfg = rvu_read64(rvu, blkaddr, 600 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 601 rvu_write64(rvu, blkaddr, 602 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 603 } 604 605 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 606 int blkaddr, int index) 607 { 608 int bank = npc_get_bank(mcam, index); 609 610 index &= (mcam->banksize - 1); 611 return rvu_read64(rvu, blkaddr, 612 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 613 } 614 615 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 616 int nixlf, u64 chan, u8 *mac_addr) 617 { 618 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 619 struct npc_install_flow_req req = { 0 }; 620 struct npc_install_flow_rsp rsp = { 0 }; 621 struct npc_mcam *mcam = &rvu->hw->mcam; 622 struct nix_rx_action action; 623 int blkaddr, index; 624 625 /* AF's VFs work in promiscuous mode */ 626 if (is_afvf(pcifunc)) 627 return; 628 629 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 630 if (blkaddr < 0) 631 return; 632 633 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 634 nixlf, NIXLF_UCAST_ENTRY); 635 636 /* Don't change the action if entry is already enabled 637 * Otherwise RSS action may get overwritten. 638 */ 639 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 640 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 641 blkaddr, index); 642 } else { 643 *(u64 *)&action = 0x00; 644 action.op = NIX_RX_ACTIONOP_UCAST; 645 action.pf_func = pcifunc; 646 } 647 648 req.default_rule = 1; 649 ether_addr_copy(req.packet.dmac, mac_addr); 650 eth_broadcast_addr((u8 *)&req.mask.dmac); 651 req.features = BIT_ULL(NPC_DMAC); 652 req.channel = chan; 653 req.intf = pfvf->nix_rx_intf; 654 req.op = action.op; 655 req.hdr.pcifunc = 0; /* AF is requester */ 656 req.vf = action.pf_func; 657 req.index = action.index; 658 req.match_id = action.match_id; 659 req.flow_key_alg = action.flow_key_alg; 660 661 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 662 } 663 664 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 665 int nixlf, u64 chan, u8 chan_cnt) 666 { 667 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 668 struct npc_install_flow_req req = { 0 }; 669 struct npc_install_flow_rsp rsp = { 0 }; 670 struct npc_mcam *mcam = &rvu->hw->mcam; 671 struct rvu_hwinfo *hw = rvu->hw; 672 int blkaddr, ucast_idx, index; 673 struct nix_rx_action action; 674 u64 relaxed_mask; 675 676 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc)) 677 return; 678 679 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 680 if (blkaddr < 0) 681 return; 682 683 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 684 nixlf, NIXLF_PROMISC_ENTRY); 685 686 if (is_cgx_vf(rvu, pcifunc)) 687 index = npc_get_nixlf_mcam_index(mcam, 688 pcifunc & ~RVU_PFVF_FUNC_MASK, 689 nixlf, NIXLF_PROMISC_ENTRY); 690 691 /* If the corresponding PF's ucast action is RSS, 692 * use the same action for promisc also 693 */ 694 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 695 nixlf, NIXLF_UCAST_ENTRY); 696 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 697 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 698 blkaddr, ucast_idx); 699 700 if (action.op != NIX_RX_ACTIONOP_RSS) { 701 *(u64 *)&action = 0x00; 702 action.op = NIX_RX_ACTIONOP_UCAST; 703 } 704 705 /* RX_ACTION set to MCAST for CGX PF's */ 706 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list && 707 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) { 708 *(u64 *)&action = 0x00; 709 action.op = NIX_RX_ACTIONOP_MCAST; 710 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 711 action.index = pfvf->promisc_mce_idx; 712 } 713 714 req.chan_mask = 0xFFFU; 715 if (chan_cnt > 1) { 716 if (!is_power_of_2(chan_cnt)) { 717 dev_err(rvu->dev, 718 "%s: channel count more than 1, must be power of 2\n", __func__); 719 return; 720 } 721 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1, 722 ilog2(chan_cnt)); 723 req.chan_mask &= relaxed_mask; 724 } 725 726 req.channel = chan; 727 req.intf = pfvf->nix_rx_intf; 728 req.entry = index; 729 req.op = action.op; 730 req.hdr.pcifunc = 0; /* AF is requester */ 731 req.vf = pcifunc; 732 req.index = action.index; 733 req.match_id = action.match_id; 734 req.flow_key_alg = action.flow_key_alg; 735 736 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 737 } 738 739 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, 740 int nixlf, bool enable) 741 { 742 struct npc_mcam *mcam = &rvu->hw->mcam; 743 int blkaddr, index; 744 745 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 746 if (blkaddr < 0) 747 return; 748 749 /* Get 'pcifunc' of PF device */ 750 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 751 752 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 753 nixlf, NIXLF_PROMISC_ENTRY); 754 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 755 } 756 757 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 758 int nixlf, u64 chan) 759 { 760 struct rvu_pfvf *pfvf; 761 struct npc_install_flow_req req = { 0 }; 762 struct npc_install_flow_rsp rsp = { 0 }; 763 struct npc_mcam *mcam = &rvu->hw->mcam; 764 struct rvu_hwinfo *hw = rvu->hw; 765 int blkaddr, index; 766 767 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 768 if (blkaddr < 0) 769 return; 770 771 /* Skip LBK VFs */ 772 if (is_afvf(pcifunc)) 773 return; 774 775 /* If pkt replication is not supported, 776 * then only PF is allowed to add a bcast match entry. 777 */ 778 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc)) 779 return; 780 781 /* Get 'pcifunc' of PF device */ 782 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 783 pfvf = rvu_get_pfvf(rvu, pcifunc); 784 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 785 nixlf, NIXLF_BCAST_ENTRY); 786 787 if (!hw->cap.nix_rx_multicast) { 788 /* Early silicon doesn't support pkt replication, 789 * so install entry with UCAST action, so that PF 790 * receives all broadcast packets. 791 */ 792 req.op = NIX_RX_ACTIONOP_UCAST; 793 } else { 794 req.op = NIX_RX_ACTIONOP_MCAST; 795 req.index = pfvf->bcast_mce_idx; 796 } 797 798 eth_broadcast_addr((u8 *)&req.packet.dmac); 799 eth_broadcast_addr((u8 *)&req.mask.dmac); 800 req.features = BIT_ULL(NPC_DMAC); 801 req.channel = chan; 802 req.intf = pfvf->nix_rx_intf; 803 req.entry = index; 804 req.hdr.pcifunc = 0; /* AF is requester */ 805 req.vf = pcifunc; 806 807 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 808 } 809 810 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 811 bool enable) 812 { 813 struct npc_mcam *mcam = &rvu->hw->mcam; 814 int blkaddr, index; 815 816 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 817 if (blkaddr < 0) 818 return; 819 820 /* Get 'pcifunc' of PF device */ 821 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 822 823 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 824 NIXLF_BCAST_ENTRY); 825 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 826 } 827 828 void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 829 u64 chan) 830 { 831 struct npc_install_flow_req req = { 0 }; 832 struct npc_install_flow_rsp rsp = { 0 }; 833 struct npc_mcam *mcam = &rvu->hw->mcam; 834 struct rvu_hwinfo *hw = rvu->hw; 835 int blkaddr, ucast_idx, index; 836 u8 mac_addr[ETH_ALEN] = { 0 }; 837 struct nix_rx_action action; 838 struct rvu_pfvf *pfvf; 839 u16 vf_func; 840 841 /* Only CGX PF/VF can add allmulticast entry */ 842 if (is_afvf(pcifunc)) 843 return; 844 845 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 846 if (blkaddr < 0) 847 return; 848 849 /* Get 'pcifunc' of PF device */ 850 vf_func = pcifunc & RVU_PFVF_FUNC_MASK; 851 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 852 pfvf = rvu_get_pfvf(rvu, pcifunc); 853 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 854 nixlf, NIXLF_ALLMULTI_ENTRY); 855 856 /* If the corresponding PF's ucast action is RSS, 857 * use the same action for multicast entry also 858 */ 859 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 860 nixlf, NIXLF_UCAST_ENTRY); 861 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 862 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 863 blkaddr, ucast_idx); 864 865 if (action.op != NIX_RX_ACTIONOP_RSS) { 866 *(u64 *)&action = 0x00; 867 action.op = NIX_RX_ACTIONOP_UCAST; 868 action.pf_func = pcifunc; 869 } 870 871 /* RX_ACTION set to MCAST for CGX PF's */ 872 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) { 873 *(u64 *)&action = 0x00; 874 action.op = NIX_RX_ACTIONOP_MCAST; 875 action.index = pfvf->mcast_mce_idx; 876 } 877 878 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */ 879 ether_addr_copy(req.packet.dmac, mac_addr); 880 ether_addr_copy(req.mask.dmac, mac_addr); 881 req.features = BIT_ULL(NPC_DMAC); 882 883 /* For cn10k the upper two bits of the channel number are 884 * cpt channel number. with masking out these bits in the 885 * mcam entry, same entry used for NIX will allow packets 886 * received from cpt for parsing. 887 */ 888 if (!is_rvu_otx2(rvu)) 889 req.chan_mask = NIX_CHAN_CPT_X2P_MASK; 890 else 891 req.chan_mask = 0xFFFU; 892 893 req.channel = chan; 894 req.intf = pfvf->nix_rx_intf; 895 req.entry = index; 896 req.op = action.op; 897 req.hdr.pcifunc = 0; /* AF is requester */ 898 req.vf = pcifunc | vf_func; 899 req.index = action.index; 900 req.match_id = action.match_id; 901 req.flow_key_alg = action.flow_key_alg; 902 903 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 904 } 905 906 void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, 907 bool enable) 908 { 909 struct npc_mcam *mcam = &rvu->hw->mcam; 910 int blkaddr, index; 911 912 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 913 if (blkaddr < 0) 914 return; 915 916 /* Get 'pcifunc' of PF device */ 917 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 918 919 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 920 NIXLF_ALLMULTI_ENTRY); 921 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 922 } 923 924 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 925 int blkaddr, u16 pcifunc, u64 rx_action) 926 { 927 int actindex, index, bank; 928 bool enable; 929 930 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 931 return; 932 933 mutex_lock(&mcam->lock); 934 for (index = 0; index < mcam->bmap_entries; index++) { 935 if (mcam->entry2target_pffunc[index] == pcifunc) { 936 bank = npc_get_bank(mcam, index); 937 actindex = index; 938 index &= (mcam->banksize - 1); 939 940 /* read vf flow entry enable status */ 941 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 942 actindex); 943 /* disable before mcam entry update */ 944 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 945 false); 946 /* update 'action' */ 947 rvu_write64(rvu, blkaddr, 948 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 949 rx_action); 950 if (enable) 951 npc_enable_mcam_entry(rvu, mcam, blkaddr, 952 actindex, true); 953 } 954 } 955 mutex_unlock(&mcam->lock); 956 } 957 958 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 959 int group, int alg_idx, int mcam_index) 960 { 961 struct npc_mcam *mcam = &rvu->hw->mcam; 962 struct rvu_hwinfo *hw = rvu->hw; 963 struct nix_rx_action action; 964 int blkaddr, index, bank; 965 struct rvu_pfvf *pfvf; 966 967 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 968 if (blkaddr < 0) 969 return; 970 971 /* Check if this is for reserved default entry */ 972 if (mcam_index < 0) { 973 if (group != DEFAULT_RSS_CONTEXT_GROUP) 974 return; 975 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 976 nixlf, NIXLF_UCAST_ENTRY); 977 } else { 978 /* TODO: validate this mcam index */ 979 index = mcam_index; 980 } 981 982 if (index >= mcam->total_entries) 983 return; 984 985 bank = npc_get_bank(mcam, index); 986 index &= (mcam->banksize - 1); 987 988 *(u64 *)&action = rvu_read64(rvu, blkaddr, 989 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 990 /* Ignore if no action was set earlier */ 991 if (!*(u64 *)&action) 992 return; 993 994 action.op = NIX_RX_ACTIONOP_RSS; 995 action.pf_func = pcifunc; 996 action.index = group; 997 action.flow_key_alg = alg_idx; 998 999 rvu_write64(rvu, blkaddr, 1000 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 1001 1002 /* update the VF flow rule action with the VF default entry action */ 1003 if (mcam_index < 0) 1004 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 1005 *(u64 *)&action); 1006 1007 /* update the action change in default rule */ 1008 pfvf = rvu_get_pfvf(rvu, pcifunc); 1009 if (pfvf->def_ucast_rule) 1010 pfvf->def_ucast_rule->rx_action = action; 1011 1012 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1013 nixlf, NIXLF_PROMISC_ENTRY); 1014 1015 /* If PF's promiscuous entry is enabled, 1016 * Set RSS action for that entry as well 1017 */ 1018 if ((!hw->cap.nix_rx_multicast || !pfvf->use_mce_list) && 1019 is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 1020 bank = npc_get_bank(mcam, index); 1021 index &= (mcam->banksize - 1); 1022 1023 rvu_write64(rvu, blkaddr, 1024 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 1025 *(u64 *)&action); 1026 } 1027 } 1028 1029 void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc, 1030 int nixlf, int type, bool enable) 1031 { 1032 struct npc_mcam *mcam = &rvu->hw->mcam; 1033 struct rvu_hwinfo *hw = rvu->hw; 1034 struct nix_mce_list *mce_list; 1035 int index, blkaddr, mce_idx; 1036 struct rvu_pfvf *pfvf; 1037 1038 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1039 if (blkaddr < 0) 1040 return; 1041 1042 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 1043 nixlf, type); 1044 1045 /* disable MCAM entry when packet replication is not supported by hw */ 1046 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) { 1047 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1048 return; 1049 } 1050 1051 /* return incase mce list is not enabled */ 1052 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 1053 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) && 1054 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list) 1055 return; 1056 1057 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 1058 1059 nix_update_mce_list(rvu, pcifunc, mce_list, 1060 mce_idx, index, enable); 1061 if (enable) 1062 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1063 } 1064 1065 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 1066 int nixlf, bool enable) 1067 { 1068 struct npc_mcam *mcam = &rvu->hw->mcam; 1069 int index, blkaddr; 1070 1071 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1072 if (blkaddr < 0) 1073 return; 1074 1075 /* Ucast MCAM match entry of this PF/VF */ 1076 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 1077 nixlf, NIXLF_UCAST_ENTRY); 1078 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 1079 1080 /* Nothing to do for VFs, on platforms where pkt replication 1081 * is not supported 1082 */ 1083 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast) 1084 return; 1085 1086 /* add/delete pf_func to broadcast MCE list */ 1087 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1088 NIXLF_BCAST_ENTRY, enable); 1089 } 1090 1091 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1092 { 1093 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 1094 1095 /* Delete multicast and promisc MCAM entries */ 1096 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1097 NIXLF_ALLMULTI_ENTRY, false); 1098 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf, 1099 NIXLF_PROMISC_ENTRY, false); 1100 } 1101 1102 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1103 { 1104 /* Enables only broadcast match entry. Promisc/Allmulti are enabled 1105 * in set_rx_mode mbox handler. 1106 */ 1107 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 1108 } 1109 1110 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1111 { 1112 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 1113 struct npc_mcam *mcam = &rvu->hw->mcam; 1114 struct rvu_npc_mcam_rule *rule, *tmp; 1115 int blkaddr; 1116 1117 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1118 if (blkaddr < 0) 1119 return; 1120 1121 mutex_lock(&mcam->lock); 1122 1123 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 1124 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1125 if (is_npc_intf_rx(rule->intf) && 1126 rule->rx_action.pf_func == pcifunc && 1127 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) { 1128 npc_enable_mcam_entry(rvu, mcam, blkaddr, 1129 rule->entry, false); 1130 rule->enable = false; 1131 /* Indicate that default rule is disabled */ 1132 if (rule->default_rule) { 1133 pfvf->def_ucast_rule = NULL; 1134 list_del(&rule->list); 1135 kfree(rule); 1136 } 1137 } 1138 } 1139 1140 mutex_unlock(&mcam->lock); 1141 1142 npc_mcam_disable_flows(rvu, pcifunc); 1143 1144 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1145 } 1146 1147 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1148 { 1149 struct npc_mcam *mcam = &rvu->hw->mcam; 1150 struct rvu_npc_mcam_rule *rule, *tmp; 1151 int blkaddr; 1152 1153 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1154 if (blkaddr < 0) 1155 return; 1156 1157 mutex_lock(&mcam->lock); 1158 1159 /* Free all MCAM entries owned by this 'pcifunc' */ 1160 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1161 1162 /* Free all MCAM counters owned by this 'pcifunc' */ 1163 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1164 1165 /* Delete MCAM entries owned by this 'pcifunc' */ 1166 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1167 if (rule->owner == pcifunc && !rule->default_rule) { 1168 list_del(&rule->list); 1169 kfree(rule); 1170 } 1171 } 1172 1173 mutex_unlock(&mcam->lock); 1174 1175 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1176 } 1177 1178 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1179 rvu_write64(rvu, blkaddr, \ 1180 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1181 1182 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1183 rvu_write64(rvu, blkaddr, \ 1184 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1185 1186 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1187 struct npc_mcam_kex *mkex, u8 intf) 1188 { 1189 int lid, lt, ld, fl; 1190 1191 if (is_npc_intf_tx(intf)) 1192 return; 1193 1194 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1195 mkex->keyx_cfg[NIX_INTF_RX]); 1196 1197 /* Program LDATA */ 1198 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1199 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1200 for (ld = 0; ld < NPC_MAX_LD; ld++) 1201 SET_KEX_LD(intf, lid, lt, ld, 1202 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1203 [lid][lt][ld]); 1204 } 1205 } 1206 /* Program LFLAGS */ 1207 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1208 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1209 SET_KEX_LDFLAGS(intf, ld, fl, 1210 mkex->intf_ld_flags[NIX_INTF_RX] 1211 [ld][fl]); 1212 } 1213 } 1214 1215 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1216 struct npc_mcam_kex *mkex, u8 intf) 1217 { 1218 int lid, lt, ld, fl; 1219 1220 if (is_npc_intf_rx(intf)) 1221 return; 1222 1223 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1224 mkex->keyx_cfg[NIX_INTF_TX]); 1225 1226 /* Program LDATA */ 1227 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1228 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1229 for (ld = 0; ld < NPC_MAX_LD; ld++) 1230 SET_KEX_LD(intf, lid, lt, ld, 1231 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1232 [lid][lt][ld]); 1233 } 1234 } 1235 /* Program LFLAGS */ 1236 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1237 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1238 SET_KEX_LDFLAGS(intf, ld, fl, 1239 mkex->intf_ld_flags[NIX_INTF_TX] 1240 [ld][fl]); 1241 } 1242 } 1243 1244 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1245 struct npc_mcam_kex *mkex) 1246 { 1247 struct rvu_hwinfo *hw = rvu->hw; 1248 u8 intf; 1249 int ld; 1250 1251 for (ld = 0; ld < NPC_MAX_LD; ld++) 1252 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1253 mkex->kex_ld_flags[ld]); 1254 1255 for (intf = 0; intf < hw->npc_intfs; intf++) { 1256 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1257 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1258 } 1259 } 1260 1261 static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, 1262 u64 *size) 1263 { 1264 u64 prfl_addr, prfl_sz; 1265 1266 if (!rvu->fwdata) 1267 return -EINVAL; 1268 1269 prfl_addr = rvu->fwdata->mcam_addr; 1270 prfl_sz = rvu->fwdata->mcam_sz; 1271 1272 if (!prfl_addr || !prfl_sz) 1273 return -EINVAL; 1274 1275 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz); 1276 if (!(*prfl_img_addr)) 1277 return -ENOMEM; 1278 1279 *size = prfl_sz; 1280 1281 return 0; 1282 } 1283 1284 /* strtoull of "mkexprof" with base:36 */ 1285 #define MKEX_END_SIGN 0xdeadbeef 1286 1287 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1288 const char *mkex_profile) 1289 { 1290 struct device *dev = &rvu->pdev->dev; 1291 struct npc_mcam_kex *mcam_kex; 1292 void __iomem *mkex_prfl_addr = NULL; 1293 u64 prfl_sz; 1294 int ret; 1295 1296 /* If user not selected mkex profile */ 1297 if (rvu->kpu_fwdata_sz || 1298 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1299 goto program_mkex; 1300 1301 /* Setting up the mapping for mkex profile image */ 1302 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); 1303 if (ret < 0) 1304 goto program_mkex; 1305 1306 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr; 1307 1308 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1309 /* Compare with mkex mod_param name string */ 1310 if (mcam_kex->mkex_sign == MKEX_SIGN && 1311 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1312 /* Due to an errata (35786) in A0/B0 pass silicon, 1313 * parse nibble enable configuration has to be 1314 * identical for both Rx and Tx interfaces. 1315 */ 1316 if (!is_rvu_96xx_B0(rvu) || 1317 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1318 rvu->kpu.mkex = mcam_kex; 1319 goto program_mkex; 1320 } 1321 1322 mcam_kex++; 1323 prfl_sz -= sizeof(struct npc_mcam_kex); 1324 } 1325 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1326 1327 program_mkex: 1328 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1329 /* Program selected mkex profile */ 1330 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1331 if (mkex_prfl_addr) 1332 iounmap(mkex_prfl_addr); 1333 } 1334 1335 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1336 const struct npc_kpu_profile_action *kpuaction, 1337 int kpu, int entry, bool pkind) 1338 { 1339 struct npc_kpu_action0 action0 = {0}; 1340 struct npc_kpu_action1 action1 = {0}; 1341 u64 reg; 1342 1343 action1.errlev = kpuaction->errlev; 1344 action1.errcode = kpuaction->errcode; 1345 action1.dp0_offset = kpuaction->dp0_offset; 1346 action1.dp1_offset = kpuaction->dp1_offset; 1347 action1.dp2_offset = kpuaction->dp2_offset; 1348 1349 if (pkind) 1350 reg = NPC_AF_PKINDX_ACTION1(entry); 1351 else 1352 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1353 1354 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1355 1356 action0.byp_count = kpuaction->bypass_count; 1357 action0.capture_ena = kpuaction->cap_ena; 1358 action0.parse_done = kpuaction->parse_done; 1359 action0.next_state = kpuaction->next_state; 1360 action0.capture_lid = kpuaction->lid; 1361 action0.capture_ltype = kpuaction->ltype; 1362 action0.capture_flags = kpuaction->flags; 1363 action0.ptr_advance = kpuaction->ptr_advance; 1364 action0.var_len_offset = kpuaction->offset; 1365 action0.var_len_mask = kpuaction->mask; 1366 action0.var_len_right = kpuaction->right; 1367 action0.var_len_shift = kpuaction->shift; 1368 1369 if (pkind) 1370 reg = NPC_AF_PKINDX_ACTION0(entry); 1371 else 1372 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1373 1374 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1375 } 1376 1377 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1378 const struct npc_kpu_profile_cam *kpucam, 1379 int kpu, int entry) 1380 { 1381 struct npc_kpu_cam cam0 = {0}; 1382 struct npc_kpu_cam cam1 = {0}; 1383 1384 cam1.state = kpucam->state & kpucam->state_mask; 1385 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1386 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1387 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1388 1389 cam0.state = ~kpucam->state & kpucam->state_mask; 1390 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1391 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1392 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1393 1394 rvu_write64(rvu, blkaddr, 1395 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1396 rvu_write64(rvu, blkaddr, 1397 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1398 } 1399 1400 static inline u64 enable_mask(int count) 1401 { 1402 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1403 } 1404 1405 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1406 const struct npc_kpu_profile *profile) 1407 { 1408 int entry, num_entries, max_entries; 1409 u64 entry_mask; 1410 1411 if (profile->cam_entries != profile->action_entries) { 1412 dev_err(rvu->dev, 1413 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1414 kpu, profile->cam_entries, profile->action_entries); 1415 } 1416 1417 max_entries = rvu->hw->npc_kpu_entries; 1418 1419 /* Program CAM match entries for previous KPU extracted data */ 1420 num_entries = min_t(int, profile->cam_entries, max_entries); 1421 for (entry = 0; entry < num_entries; entry++) 1422 npc_config_kpucam(rvu, blkaddr, 1423 &profile->cam[entry], kpu, entry); 1424 1425 /* Program this KPU's actions */ 1426 num_entries = min_t(int, profile->action_entries, max_entries); 1427 for (entry = 0; entry < num_entries; entry++) 1428 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1429 kpu, entry, false); 1430 1431 /* Enable all programmed entries */ 1432 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1433 entry_mask = enable_mask(num_entries); 1434 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ 1435 if (!rvu->kpu.custom) 1436 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); 1437 rvu_write64(rvu, blkaddr, 1438 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask); 1439 if (num_entries > 64) { 1440 rvu_write64(rvu, blkaddr, 1441 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1442 enable_mask(num_entries - 64)); 1443 } 1444 1445 /* Enable this KPU */ 1446 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1447 } 1448 1449 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1450 { 1451 profile->custom = 0; 1452 profile->name = def_pfl_name; 1453 profile->version = NPC_KPU_PROFILE_VER; 1454 profile->ikpu = ikpu_action_entries; 1455 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1456 profile->kpu = npc_kpu_profiles; 1457 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1458 profile->lt_def = &npc_lt_defaults; 1459 profile->mkex = &npc_mkex_default; 1460 1461 return 0; 1462 } 1463 1464 static int npc_apply_custom_kpu(struct rvu *rvu, 1465 struct npc_kpu_profile_adapter *profile) 1466 { 1467 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; 1468 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; 1469 struct npc_kpu_profile_action *action; 1470 struct npc_kpu_profile_cam *cam; 1471 struct npc_kpu_fwdata *fw_kpu; 1472 int entries; 1473 u16 kpu, entry; 1474 1475 if (rvu->kpu_fwdata_sz < hdr_sz) { 1476 dev_warn(rvu->dev, "Invalid KPU profile size\n"); 1477 return -EINVAL; 1478 } 1479 if (le64_to_cpu(fw->signature) != KPU_SIGN) { 1480 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", 1481 fw->signature); 1482 return -EINVAL; 1483 } 1484 /* Verify if the using known profile structure */ 1485 if (NPC_KPU_VER_MAJ(profile->version) > 1486 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { 1487 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", 1488 NPC_KPU_VER_MAJ(profile->version), 1489 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); 1490 return -EINVAL; 1491 } 1492 /* Verify if profile is aligned with the required kernel changes */ 1493 if (NPC_KPU_VER_MIN(profile->version) < 1494 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { 1495 dev_warn(rvu->dev, 1496 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", 1497 NPC_KPU_VER_MAJ(profile->version), 1498 NPC_KPU_VER_MIN(profile->version), 1499 NPC_KPU_VER_PATCH(profile->version), 1500 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), 1501 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), 1502 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); 1503 return -EINVAL; 1504 } 1505 /* Verify if profile fits the HW */ 1506 if (fw->kpus > profile->kpus) { 1507 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, 1508 profile->kpus); 1509 return -EINVAL; 1510 } 1511 1512 profile->custom = 1; 1513 profile->name = fw->name; 1514 profile->version = le64_to_cpu(fw->version); 1515 profile->mkex = &fw->mkex; 1516 profile->lt_def = &fw->lt_def; 1517 1518 for (kpu = 0; kpu < fw->kpus; kpu++) { 1519 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); 1520 if (fw_kpu->entries > KPU_MAX_CST_ENT) 1521 dev_warn(rvu->dev, 1522 "Too many custom entries on KPU%d: %d > %d\n", 1523 kpu, fw_kpu->entries, KPU_MAX_CST_ENT); 1524 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT); 1525 cam = (struct npc_kpu_profile_cam *)fw_kpu->data; 1526 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); 1527 action = (struct npc_kpu_profile_action *)(fw->data + offset); 1528 offset += fw_kpu->entries * sizeof(*action); 1529 if (rvu->kpu_fwdata_sz < hdr_sz + offset) { 1530 dev_warn(rvu->dev, 1531 "Profile size mismatch on KPU%i parsing.\n", 1532 kpu + 1); 1533 return -EINVAL; 1534 } 1535 for (entry = 0; entry < entries; entry++) { 1536 profile->kpu[kpu].cam[entry] = cam[entry]; 1537 profile->kpu[kpu].action[entry] = action[entry]; 1538 } 1539 } 1540 1541 return 0; 1542 } 1543 1544 static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr, 1545 u64 prfl_sz, const char *kpu_profile) 1546 { 1547 struct npc_kpu_profile_fwdata *kpu_data = NULL; 1548 int rc = -EINVAL; 1549 1550 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr; 1551 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN && 1552 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) { 1553 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n", 1554 kpu_profile); 1555 rvu->kpu_fwdata = kpu_data; 1556 rvu->kpu_fwdata_sz = prfl_sz; 1557 rvu->kpu_prfl_addr = prfl_addr; 1558 rc = 0; 1559 } 1560 1561 return rc; 1562 } 1563 1564 static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz, 1565 const char *kpu_profile) 1566 { 1567 struct npc_coalesced_kpu_prfl *img_data = NULL; 1568 int i = 0, rc = -EINVAL; 1569 void __iomem *kpu_prfl_addr; 1570 u16 offset; 1571 1572 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr; 1573 if (le64_to_cpu(img_data->signature) == KPU_SIGN && 1574 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) { 1575 /* Loaded profile is a single KPU profile. */ 1576 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr, 1577 prfl_sz, kpu_profile); 1578 goto done; 1579 } 1580 1581 /* Loaded profile is coalesced image, offset of first KPU profile.*/ 1582 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) + 1583 (img_data->num_prfl * sizeof(uint16_t)); 1584 /* Check if mapped image is coalesced image. */ 1585 while (i < img_data->num_prfl) { 1586 /* Profile image offsets are rounded up to next 8 multiple.*/ 1587 offset = ALIGN_8B_CEIL(offset); 1588 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr + 1589 offset); 1590 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr, 1591 img_data->prfl_sz[i], kpu_profile); 1592 if (!rc) 1593 break; 1594 /* Calculating offset of profile image based on profile size.*/ 1595 offset += img_data->prfl_sz[i]; 1596 i++; 1597 } 1598 done: 1599 return rc; 1600 } 1601 1602 static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile) 1603 { 1604 int ret = -EINVAL; 1605 u64 prfl_sz; 1606 1607 /* Setting up the mapping for NPC profile image */ 1608 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz); 1609 if (ret < 0) 1610 goto done; 1611 1612 /* Detect if profile is coalesced or single KPU profile and load */ 1613 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile); 1614 if (ret == 0) 1615 goto done; 1616 1617 /* Cleaning up if KPU profile image from fwdata is not valid. */ 1618 if (rvu->kpu_prfl_addr) { 1619 iounmap(rvu->kpu_prfl_addr); 1620 rvu->kpu_prfl_addr = NULL; 1621 rvu->kpu_fwdata_sz = 0; 1622 rvu->kpu_fwdata = NULL; 1623 } 1624 1625 done: 1626 return ret; 1627 } 1628 1629 static void npc_load_kpu_profile(struct rvu *rvu) 1630 { 1631 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1632 const char *kpu_profile = rvu->kpu_pfl_name; 1633 const struct firmware *fw = NULL; 1634 bool retry_fwdb = false; 1635 1636 /* If user not specified profile customization */ 1637 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) 1638 goto revert_to_default; 1639 /* First prepare default KPU, then we'll customize top entries. */ 1640 npc_prepare_default_kpu(profile); 1641 1642 /* Order of preceedence for load loading NPC profile (high to low) 1643 * Firmware binary in filesystem. 1644 * Firmware database method. 1645 * Default KPU profile. 1646 */ 1647 if (!request_firmware(&fw, kpu_profile, rvu->dev)) { 1648 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n", 1649 kpu_profile); 1650 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL); 1651 if (rvu->kpu_fwdata) { 1652 memcpy(rvu->kpu_fwdata, fw->data, fw->size); 1653 rvu->kpu_fwdata_sz = fw->size; 1654 } 1655 release_firmware(fw); 1656 retry_fwdb = true; 1657 goto program_kpu; 1658 } 1659 1660 load_image_fwdb: 1661 /* Loading the KPU profile using firmware database */ 1662 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile)) 1663 goto revert_to_default; 1664 1665 program_kpu: 1666 /* Apply profile customization if firmware was loaded. */ 1667 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) { 1668 /* If image from firmware filesystem fails to load or invalid 1669 * retry with firmware database method. 1670 */ 1671 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) { 1672 /* Loading image from firmware database failed. */ 1673 if (rvu->kpu_prfl_addr) { 1674 iounmap(rvu->kpu_prfl_addr); 1675 rvu->kpu_prfl_addr = NULL; 1676 } else { 1677 kfree(rvu->kpu_fwdata); 1678 } 1679 rvu->kpu_fwdata = NULL; 1680 rvu->kpu_fwdata_sz = 0; 1681 if (retry_fwdb) { 1682 retry_fwdb = false; 1683 goto load_image_fwdb; 1684 } 1685 } 1686 1687 dev_warn(rvu->dev, 1688 "Can't load KPU profile %s. Using default.\n", 1689 kpu_profile); 1690 kfree(rvu->kpu_fwdata); 1691 rvu->kpu_fwdata = NULL; 1692 goto revert_to_default; 1693 } 1694 1695 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n", 1696 profile->name, NPC_KPU_VER_MAJ(profile->version), 1697 NPC_KPU_VER_MIN(profile->version), 1698 NPC_KPU_VER_PATCH(profile->version)); 1699 1700 return; 1701 1702 revert_to_default: 1703 npc_prepare_default_kpu(profile); 1704 } 1705 1706 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1707 { 1708 struct rvu_hwinfo *hw = rvu->hw; 1709 int num_pkinds, num_kpus, idx; 1710 struct npc_pkind *pkind; 1711 1712 /* Disable all KPUs and their entries */ 1713 for (idx = 0; idx < hw->npc_kpus; idx++) { 1714 rvu_write64(rvu, blkaddr, 1715 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1716 rvu_write64(rvu, blkaddr, 1717 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1718 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1719 } 1720 1721 /* Load and customize KPU profile. */ 1722 npc_load_kpu_profile(rvu); 1723 1724 /* First program IKPU profile i.e PKIND configs. 1725 * Check HW max count to avoid configuring junk or 1726 * writing to unsupported CSR addresses. 1727 */ 1728 pkind = &hw->pkind; 1729 num_pkinds = rvu->kpu.pkinds; 1730 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 1731 1732 for (idx = 0; idx < num_pkinds; idx++) 1733 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1734 1735 /* Program KPU CAM and Action profiles */ 1736 num_kpus = rvu->kpu.kpus; 1737 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1738 1739 for (idx = 0; idx < num_kpus; idx++) 1740 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1741 } 1742 1743 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1744 { 1745 int nixlf_count = rvu_get_nixlf_count(rvu); 1746 struct npc_mcam *mcam = &rvu->hw->mcam; 1747 int rsvd, err; 1748 u64 cfg; 1749 1750 /* Actual number of MCAM entries vary by entry size */ 1751 cfg = (rvu_read64(rvu, blkaddr, 1752 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1753 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1754 mcam->keysize = cfg; 1755 1756 /* Number of banks combined per MCAM entry */ 1757 if (cfg == NPC_MCAM_KEY_X4) 1758 mcam->banks_per_entry = 4; 1759 else if (cfg == NPC_MCAM_KEY_X2) 1760 mcam->banks_per_entry = 2; 1761 else 1762 mcam->banks_per_entry = 1; 1763 1764 /* Reserve one MCAM entry for each of the NIX LF to 1765 * guarantee space to install default matching DMAC rule. 1766 * Also reserve 2 MCAM entries for each PF for default 1767 * channel based matching or 'bcast & promisc' matching to 1768 * support BCAST and PROMISC modes of operation for PFs. 1769 * PF0 is excluded. 1770 */ 1771 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1772 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1773 if (mcam->total_entries <= rsvd) { 1774 dev_warn(rvu->dev, 1775 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1776 mcam->total_entries); 1777 return -ENOMEM; 1778 } 1779 1780 mcam->bmap_entries = mcam->total_entries - rsvd; 1781 mcam->nixlf_offset = mcam->bmap_entries; 1782 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1783 1784 /* Allocate bitmaps for managing MCAM entries */ 1785 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1786 sizeof(long), GFP_KERNEL); 1787 if (!mcam->bmap) 1788 return -ENOMEM; 1789 1790 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1791 BITS_TO_LONGS(mcam->bmap_entries), 1792 sizeof(long), GFP_KERNEL); 1793 if (!mcam->bmap_reverse) 1794 return -ENOMEM; 1795 1796 mcam->bmap_fcnt = mcam->bmap_entries; 1797 1798 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1799 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1800 sizeof(u16), GFP_KERNEL); 1801 if (!mcam->entry2pfvf_map) 1802 return -ENOMEM; 1803 1804 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1805 * allocations and another 1/8th at the top for high priority 1806 * allocations. 1807 */ 1808 mcam->lprio_count = mcam->bmap_entries / 8; 1809 if (mcam->lprio_count > BITS_PER_LONG) 1810 mcam->lprio_count = round_down(mcam->lprio_count, 1811 BITS_PER_LONG); 1812 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1813 mcam->hprio_count = mcam->lprio_count; 1814 mcam->hprio_end = mcam->hprio_count; 1815 1816 1817 /* Allocate bitmap for managing MCAM counters and memory 1818 * for saving counter to RVU PFFUNC allocation mapping. 1819 */ 1820 err = rvu_alloc_bitmap(&mcam->counters); 1821 if (err) 1822 return err; 1823 1824 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1825 sizeof(u16), GFP_KERNEL); 1826 if (!mcam->cntr2pfvf_map) 1827 goto free_mem; 1828 1829 /* Alloc memory for MCAM entry to counter mapping and for tracking 1830 * counter's reference count. 1831 */ 1832 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1833 sizeof(u16), GFP_KERNEL); 1834 if (!mcam->entry2cntr_map) 1835 goto free_mem; 1836 1837 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1838 sizeof(u16), GFP_KERNEL); 1839 if (!mcam->cntr_refcnt) 1840 goto free_mem; 1841 1842 /* Alloc memory for saving target device of mcam rule */ 1843 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1844 sizeof(u16), GFP_KERNEL); 1845 if (!mcam->entry2target_pffunc) 1846 goto free_mem; 1847 1848 mutex_init(&mcam->lock); 1849 1850 return 0; 1851 1852 free_mem: 1853 kfree(mcam->counters.bmap); 1854 return -ENOMEM; 1855 } 1856 1857 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1858 { 1859 struct npc_pkind *pkind = &rvu->hw->pkind; 1860 struct npc_mcam *mcam = &rvu->hw->mcam; 1861 struct rvu_hwinfo *hw = rvu->hw; 1862 u64 npc_const, npc_const1; 1863 u64 npc_const2 = 0; 1864 1865 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1866 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1867 if (npc_const1 & BIT_ULL(63)) 1868 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1869 1870 pkind->rsrc.max = (npc_const1 >> 12) & 0xFFULL; 1871 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1872 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1873 hw->npc_intfs = npc_const & 0xFULL; 1874 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1875 1876 mcam->banks = (npc_const >> 44) & 0xFULL; 1877 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1878 /* Extended set */ 1879 if (npc_const2) { 1880 hw->npc_ext_set = true; 1881 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1882 mcam->banksize = npc_const2 & 0xFFFFULL; 1883 } 1884 1885 mcam->counters.max = hw->npc_counters; 1886 } 1887 1888 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1889 { 1890 struct npc_mcam *mcam = &rvu->hw->mcam; 1891 struct rvu_hwinfo *hw = rvu->hw; 1892 u64 nibble_ena, rx_kex, tx_kex; 1893 u8 intf; 1894 1895 /* Reserve last counter for MCAM RX miss action which is set to 1896 * drop packet. This way we will know how many pkts didn't match 1897 * any MCAM entry. 1898 */ 1899 mcam->counters.max--; 1900 mcam->rx_miss_act_cntr = mcam->counters.max; 1901 1902 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1903 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1904 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1905 1906 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1907 if (nibble_ena) { 1908 tx_kex &= ~NPC_PARSE_NIBBLE; 1909 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1910 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1911 } 1912 1913 /* Configure RX interfaces */ 1914 for (intf = 0; intf < hw->npc_intfs; intf++) { 1915 if (is_npc_intf_tx(intf)) 1916 continue; 1917 1918 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1919 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1920 rx_kex); 1921 1922 /* If MCAM lookup doesn't result in a match, drop the received 1923 * packet. And map this action to a counter to count dropped 1924 * packets. 1925 */ 1926 rvu_write64(rvu, blkaddr, 1927 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1928 1929 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1930 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1931 */ 1932 rvu_write64(rvu, blkaddr, 1933 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1934 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1935 BIT_ULL(9) | mcam->rx_miss_act_cntr); 1936 } 1937 1938 /* Configure TX interfaces */ 1939 for (intf = 0; intf < hw->npc_intfs; intf++) { 1940 if (is_npc_intf_rx(intf)) 1941 continue; 1942 1943 /* Extract Ltypes LID_LA to LID_LE */ 1944 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1945 tx_kex); 1946 1947 /* Set TX miss action to UCAST_DEFAULT i.e 1948 * transmit the packet on NIX LF SQ's default channel. 1949 */ 1950 rvu_write64(rvu, blkaddr, 1951 NPC_AF_INTFX_MISS_ACT(intf), 1952 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1953 } 1954 } 1955 1956 int rvu_npc_init(struct rvu *rvu) 1957 { 1958 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1959 struct npc_pkind *pkind = &rvu->hw->pkind; 1960 struct npc_mcam *mcam = &rvu->hw->mcam; 1961 int blkaddr, entry, bank, err; 1962 1963 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1964 if (blkaddr < 0) { 1965 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1966 return -ENODEV; 1967 } 1968 1969 rvu_npc_hw_init(rvu, blkaddr); 1970 1971 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1972 for (bank = 0; bank < mcam->banks; bank++) { 1973 for (entry = 0; entry < mcam->banksize; entry++) 1974 rvu_write64(rvu, blkaddr, 1975 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1976 } 1977 1978 err = rvu_alloc_bitmap(&pkind->rsrc); 1979 if (err) 1980 return err; 1981 1982 /* Allocate mem for pkind to PF and channel mapping info */ 1983 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 1984 sizeof(u32), GFP_KERNEL); 1985 if (!pkind->pfchan_map) 1986 return -ENOMEM; 1987 1988 /* Configure KPU profile */ 1989 npc_parser_profile_init(rvu, blkaddr); 1990 1991 /* Config Outer L2, IPv4's NPC layer info */ 1992 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1993 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 1994 kpu->lt_def->pck_ol2.ltype_mask); 1995 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1996 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 1997 kpu->lt_def->pck_oip4.ltype_mask); 1998 1999 /* Config Inner IPV4 NPC layer info */ 2000 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 2001 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 2002 kpu->lt_def->pck_iip4.ltype_mask); 2003 2004 /* Enable below for Rx pkts. 2005 * - Outer IPv4 header checksum validation. 2006 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 2007 * - Inner IPv4 header checksum validation. 2008 * - Set non zero checksum error code value 2009 */ 2010 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 2011 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 2012 BIT_ULL(32) | BIT_ULL(24) | BIT_ULL(6) | 2013 BIT_ULL(2) | BIT_ULL(1)); 2014 2015 rvu_npc_setup_interfaces(rvu, blkaddr); 2016 2017 /* Configure MKEX profile */ 2018 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 2019 2020 err = npc_mcam_rsrcs_init(rvu, blkaddr); 2021 if (err) 2022 return err; 2023 2024 err = npc_flow_steering_init(rvu, blkaddr); 2025 if (err) { 2026 dev_err(rvu->dev, 2027 "Incorrect mkex profile loaded using default mkex\n"); 2028 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 2029 } 2030 2031 return 0; 2032 } 2033 2034 void rvu_npc_freemem(struct rvu *rvu) 2035 { 2036 struct npc_pkind *pkind = &rvu->hw->pkind; 2037 struct npc_mcam *mcam = &rvu->hw->mcam; 2038 2039 kfree(pkind->rsrc.bmap); 2040 kfree(mcam->counters.bmap); 2041 if (rvu->kpu_prfl_addr) 2042 iounmap(rvu->kpu_prfl_addr); 2043 else 2044 kfree(rvu->kpu_fwdata); 2045 mutex_destroy(&mcam->lock); 2046 } 2047 2048 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 2049 int blkaddr, int *alloc_cnt, 2050 int *enable_cnt) 2051 { 2052 struct npc_mcam *mcam = &rvu->hw->mcam; 2053 int entry; 2054 2055 *alloc_cnt = 0; 2056 *enable_cnt = 0; 2057 2058 for (entry = 0; entry < mcam->bmap_entries; entry++) { 2059 if (mcam->entry2pfvf_map[entry] == pcifunc) { 2060 (*alloc_cnt)++; 2061 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 2062 (*enable_cnt)++; 2063 } 2064 } 2065 } 2066 2067 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 2068 int blkaddr, int *alloc_cnt, 2069 int *enable_cnt) 2070 { 2071 struct npc_mcam *mcam = &rvu->hw->mcam; 2072 int cntr; 2073 2074 *alloc_cnt = 0; 2075 *enable_cnt = 0; 2076 2077 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2078 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2079 (*alloc_cnt)++; 2080 if (mcam->cntr_refcnt[cntr]) 2081 (*enable_cnt)++; 2082 } 2083 } 2084 } 2085 2086 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 2087 u16 pcifunc, int entry) 2088 { 2089 /* verify AF installed entries */ 2090 if (is_pffunc_af(pcifunc)) 2091 return 0; 2092 /* Verify if entry is valid and if it is indeed 2093 * allocated to the requesting PFFUNC. 2094 */ 2095 if (entry >= mcam->bmap_entries) 2096 return NPC_MCAM_INVALID_REQ; 2097 2098 if (pcifunc != mcam->entry2pfvf_map[entry]) 2099 return NPC_MCAM_PERM_DENIED; 2100 2101 return 0; 2102 } 2103 2104 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 2105 u16 pcifunc, int cntr) 2106 { 2107 /* Verify if counter is valid and if it is indeed 2108 * allocated to the requesting PFFUNC. 2109 */ 2110 if (cntr >= mcam->counters.max) 2111 return NPC_MCAM_INVALID_REQ; 2112 2113 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 2114 return NPC_MCAM_PERM_DENIED; 2115 2116 return 0; 2117 } 2118 2119 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 2120 int blkaddr, u16 entry, u16 cntr) 2121 { 2122 u16 index = entry & (mcam->banksize - 1); 2123 u16 bank = npc_get_bank(mcam, entry); 2124 2125 /* Set mapping and increment counter's refcnt */ 2126 mcam->entry2cntr_map[entry] = cntr; 2127 mcam->cntr_refcnt[cntr]++; 2128 /* Enable stats 2129 * NPC_AF_MCAMEX_BANKX_STAT_ACT[14:12] - counter[11:9] 2130 * NPC_AF_MCAMEX_BANKX_STAT_ACT[8:0] - counter[8:0] 2131 */ 2132 rvu_write64(rvu, blkaddr, 2133 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 2134 ((cntr >> 9) << 12) | BIT_ULL(9) | cntr); 2135 } 2136 2137 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 2138 struct npc_mcam *mcam, 2139 int blkaddr, u16 entry, u16 cntr) 2140 { 2141 u16 index = entry & (mcam->banksize - 1); 2142 u16 bank = npc_get_bank(mcam, entry); 2143 2144 /* Remove mapping and reduce counter's refcnt */ 2145 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 2146 mcam->cntr_refcnt[cntr]--; 2147 /* Disable stats */ 2148 rvu_write64(rvu, blkaddr, 2149 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 2150 } 2151 2152 /* Sets MCAM entry in bitmap as used. Update 2153 * reverse bitmap too. Should be called with 2154 * 'mcam->lock' held. 2155 */ 2156 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 2157 { 2158 u16 entry, rentry; 2159 2160 entry = index; 2161 rentry = mcam->bmap_entries - index - 1; 2162 2163 __set_bit(entry, mcam->bmap); 2164 __set_bit(rentry, mcam->bmap_reverse); 2165 mcam->bmap_fcnt--; 2166 } 2167 2168 /* Sets MCAM entry in bitmap as free. Update 2169 * reverse bitmap too. Should be called with 2170 * 'mcam->lock' held. 2171 */ 2172 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 2173 { 2174 u16 entry, rentry; 2175 2176 entry = index; 2177 rentry = mcam->bmap_entries - index - 1; 2178 2179 __clear_bit(entry, mcam->bmap); 2180 __clear_bit(rentry, mcam->bmap_reverse); 2181 mcam->bmap_fcnt++; 2182 } 2183 2184 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 2185 int blkaddr, u16 pcifunc) 2186 { 2187 u16 index, cntr; 2188 2189 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 2190 for (index = 0; index < mcam->bmap_entries; index++) { 2191 if (mcam->entry2pfvf_map[index] == pcifunc) { 2192 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 2193 /* Free the entry in bitmap */ 2194 npc_mcam_clear_bit(mcam, index); 2195 /* Disable the entry */ 2196 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 2197 2198 /* Update entry2counter mapping */ 2199 cntr = mcam->entry2cntr_map[index]; 2200 if (cntr != NPC_MCAM_INVALID_MAP) 2201 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 2202 blkaddr, index, 2203 cntr); 2204 mcam->entry2target_pffunc[index] = 0x0; 2205 } 2206 } 2207 } 2208 2209 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 2210 u16 pcifunc) 2211 { 2212 u16 cntr; 2213 2214 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 2215 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 2216 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 2217 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 2218 mcam->cntr_refcnt[cntr] = 0; 2219 rvu_free_rsrc(&mcam->counters, cntr); 2220 /* This API is expected to be called after freeing 2221 * MCAM entries, which inturn will remove 2222 * 'entry to counter' mapping. 2223 * No need to do it again. 2224 */ 2225 } 2226 } 2227 } 2228 2229 /* Find area of contiguous free entries of size 'nr'. 2230 * If not found return max contiguous free entries available. 2231 */ 2232 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 2233 u16 nr, u16 *max_area) 2234 { 2235 u16 max_area_start = 0; 2236 u16 index, next, end; 2237 2238 *max_area = 0; 2239 2240 again: 2241 index = find_next_zero_bit(map, size, start); 2242 if (index >= size) 2243 return max_area_start; 2244 2245 end = ((index + nr) >= size) ? size : index + nr; 2246 next = find_next_bit(map, end, index); 2247 if (*max_area < (next - index)) { 2248 *max_area = next - index; 2249 max_area_start = index; 2250 } 2251 2252 if (next < end) { 2253 start = next + 1; 2254 goto again; 2255 } 2256 2257 return max_area_start; 2258 } 2259 2260 /* Find number of free MCAM entries available 2261 * within range i.e in between 'start' and 'end'. 2262 */ 2263 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 2264 { 2265 u16 index, next; 2266 u16 fcnt = 0; 2267 2268 again: 2269 if (start >= end) 2270 return fcnt; 2271 2272 index = find_next_zero_bit(map, end, start); 2273 if (index >= end) 2274 return fcnt; 2275 2276 next = find_next_bit(map, end, index); 2277 if (next <= end) { 2278 fcnt += next - index; 2279 start = next + 1; 2280 goto again; 2281 } 2282 2283 fcnt += end - index; 2284 return fcnt; 2285 } 2286 2287 static void 2288 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 2289 struct npc_mcam_alloc_entry_req *req, 2290 u16 *start, u16 *end, bool *reverse) 2291 { 2292 u16 fcnt; 2293 2294 if (req->priority == NPC_MCAM_HIGHER_PRIO) 2295 goto hprio; 2296 2297 /* For a low priority entry allocation 2298 * - If reference entry is not in hprio zone then 2299 * search range: ref_entry to end. 2300 * - If reference entry is in hprio zone and if 2301 * request can be accomodated in non-hprio zone then 2302 * search range: 'start of middle zone' to 'end' 2303 * - else search in reverse, so that less number of hprio 2304 * zone entries are allocated. 2305 */ 2306 2307 *reverse = false; 2308 *start = req->ref_entry + 1; 2309 *end = mcam->bmap_entries; 2310 2311 if (req->ref_entry >= mcam->hprio_end) 2312 return; 2313 2314 fcnt = npc_mcam_get_free_count(mcam->bmap, 2315 mcam->hprio_end, mcam->bmap_entries); 2316 if (fcnt > req->count) 2317 *start = mcam->hprio_end; 2318 else 2319 *reverse = true; 2320 return; 2321 2322 hprio: 2323 /* For a high priority entry allocation, search is always 2324 * in reverse to preserve hprio zone entries. 2325 * - If reference entry is not in lprio zone then 2326 * search range: 0 to ref_entry. 2327 * - If reference entry is in lprio zone and if 2328 * request can be accomodated in middle zone then 2329 * search range: 'hprio_end' to 'lprio_start' 2330 */ 2331 2332 *reverse = true; 2333 *start = 0; 2334 *end = req->ref_entry; 2335 2336 if (req->ref_entry <= mcam->lprio_start) 2337 return; 2338 2339 fcnt = npc_mcam_get_free_count(mcam->bmap, 2340 mcam->hprio_end, mcam->lprio_start); 2341 if (fcnt < req->count) 2342 return; 2343 *start = mcam->hprio_end; 2344 *end = mcam->lprio_start; 2345 } 2346 2347 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 2348 struct npc_mcam_alloc_entry_req *req, 2349 struct npc_mcam_alloc_entry_rsp *rsp) 2350 { 2351 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 2352 u16 fcnt, hp_fcnt, lp_fcnt; 2353 u16 start, end, index; 2354 int entry, next_start; 2355 bool reverse = false; 2356 unsigned long *bmap; 2357 u16 max_contig; 2358 2359 mutex_lock(&mcam->lock); 2360 2361 /* Check if there are any free entries */ 2362 if (!mcam->bmap_fcnt) { 2363 mutex_unlock(&mcam->lock); 2364 return NPC_MCAM_ALLOC_FAILED; 2365 } 2366 2367 /* MCAM entries are divided into high priority, middle and 2368 * low priority zones. Idea is to not allocate top and lower 2369 * most entries as much as possible, this is to increase 2370 * probability of honouring priority allocation requests. 2371 * 2372 * Two bitmaps are used for mcam entry management, 2373 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 2374 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 2375 * 2376 * Reverse bitmap is used to allocate entries 2377 * - when a higher priority entry is requested 2378 * - when available free entries are less. 2379 * Lower priority ones out of avaialble free entries are always 2380 * chosen when 'high vs low' question arises. 2381 */ 2382 2383 /* Get the search range for priority allocation request */ 2384 if (req->priority) { 2385 npc_get_mcam_search_range_priority(mcam, req, 2386 &start, &end, &reverse); 2387 goto alloc; 2388 } 2389 2390 /* Find out the search range for non-priority allocation request 2391 * 2392 * Get MCAM free entry count in middle zone. 2393 */ 2394 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 2395 mcam->lprio_start, 2396 mcam->bmap_entries); 2397 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 2398 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 2399 2400 /* Check if request can be accomodated in the middle zone */ 2401 if (fcnt > req->count) { 2402 start = mcam->hprio_end; 2403 end = mcam->lprio_start; 2404 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 2405 /* Expand search zone from half of hprio zone to 2406 * half of lprio zone. 2407 */ 2408 start = mcam->hprio_end / 2; 2409 end = mcam->bmap_entries - (mcam->lprio_count / 2); 2410 reverse = true; 2411 } else { 2412 /* Not enough free entries, search all entries in reverse, 2413 * so that low priority ones will get used up. 2414 */ 2415 reverse = true; 2416 start = 0; 2417 end = mcam->bmap_entries; 2418 } 2419 2420 alloc: 2421 if (reverse) { 2422 bmap = mcam->bmap_reverse; 2423 start = mcam->bmap_entries - start; 2424 end = mcam->bmap_entries - end; 2425 index = start; 2426 start = end; 2427 end = index; 2428 } else { 2429 bmap = mcam->bmap; 2430 } 2431 2432 if (req->contig) { 2433 /* Allocate requested number of contiguous entries, if 2434 * unsuccessful find max contiguous entries available. 2435 */ 2436 index = npc_mcam_find_zero_area(bmap, end, start, 2437 req->count, &max_contig); 2438 rsp->count = max_contig; 2439 if (reverse) 2440 rsp->entry = mcam->bmap_entries - index - max_contig; 2441 else 2442 rsp->entry = index; 2443 } else { 2444 /* Allocate requested number of non-contiguous entries, 2445 * if unsuccessful allocate as many as possible. 2446 */ 2447 rsp->count = 0; 2448 next_start = start; 2449 for (entry = 0; entry < req->count; entry++) { 2450 index = find_next_zero_bit(bmap, end, next_start); 2451 if (index >= end) 2452 break; 2453 2454 next_start = start + (index - start) + 1; 2455 2456 /* Save the entry's index */ 2457 if (reverse) 2458 index = mcam->bmap_entries - index - 1; 2459 entry_list[entry] = index; 2460 rsp->count++; 2461 } 2462 } 2463 2464 /* If allocating requested no of entries is unsucessful, 2465 * expand the search range to full bitmap length and retry. 2466 */ 2467 if (!req->priority && (rsp->count < req->count) && 2468 ((end - start) != mcam->bmap_entries)) { 2469 reverse = true; 2470 start = 0; 2471 end = mcam->bmap_entries; 2472 goto alloc; 2473 } 2474 2475 /* For priority entry allocation requests, if allocation is 2476 * failed then expand search to max possible range and retry. 2477 */ 2478 if (req->priority && rsp->count < req->count) { 2479 if (req->priority == NPC_MCAM_LOWER_PRIO && 2480 (start != (req->ref_entry + 1))) { 2481 start = req->ref_entry + 1; 2482 end = mcam->bmap_entries; 2483 reverse = false; 2484 goto alloc; 2485 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2486 ((end - start) != req->ref_entry)) { 2487 start = 0; 2488 end = req->ref_entry; 2489 reverse = true; 2490 goto alloc; 2491 } 2492 } 2493 2494 /* Copy MCAM entry indices into mbox response entry_list. 2495 * Requester always expects indices in ascending order, so 2496 * so reverse the list if reverse bitmap is used for allocation. 2497 */ 2498 if (!req->contig && rsp->count) { 2499 index = 0; 2500 for (entry = rsp->count - 1; entry >= 0; entry--) { 2501 if (reverse) 2502 rsp->entry_list[index++] = entry_list[entry]; 2503 else 2504 rsp->entry_list[entry] = entry_list[entry]; 2505 } 2506 } 2507 2508 /* Mark the allocated entries as used and set nixlf mapping */ 2509 for (entry = 0; entry < rsp->count; entry++) { 2510 index = req->contig ? 2511 (rsp->entry + entry) : rsp->entry_list[entry]; 2512 npc_mcam_set_bit(mcam, index); 2513 mcam->entry2pfvf_map[index] = pcifunc; 2514 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2515 } 2516 2517 /* Update available free count in mbox response */ 2518 rsp->free_count = mcam->bmap_fcnt; 2519 2520 mutex_unlock(&mcam->lock); 2521 return 0; 2522 } 2523 2524 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2525 struct npc_mcam_alloc_entry_req *req, 2526 struct npc_mcam_alloc_entry_rsp *rsp) 2527 { 2528 struct npc_mcam *mcam = &rvu->hw->mcam; 2529 u16 pcifunc = req->hdr.pcifunc; 2530 int blkaddr; 2531 2532 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2533 if (blkaddr < 0) 2534 return NPC_MCAM_INVALID_REQ; 2535 2536 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2537 rsp->free_count = 0; 2538 2539 /* Check if ref_entry is within range */ 2540 if (req->priority && req->ref_entry >= mcam->bmap_entries) { 2541 dev_err(rvu->dev, "%s: reference entry %d is out of range\n", 2542 __func__, req->ref_entry); 2543 return NPC_MCAM_INVALID_REQ; 2544 } 2545 2546 /* ref_entry can't be '0' if requested priority is high. 2547 * Can't be last entry if requested priority is low. 2548 */ 2549 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2550 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2551 req->priority == NPC_MCAM_LOWER_PRIO)) 2552 return NPC_MCAM_INVALID_REQ; 2553 2554 /* Since list of allocated indices needs to be sent to requester, 2555 * max number of non-contiguous entries per mbox msg is limited. 2556 */ 2557 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) { 2558 dev_err(rvu->dev, 2559 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n", 2560 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES); 2561 return NPC_MCAM_INVALID_REQ; 2562 } 2563 2564 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2565 if (!is_nixlf_attached(rvu, pcifunc)) 2566 return NPC_MCAM_ALLOC_DENIED; 2567 2568 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2569 } 2570 2571 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2572 struct npc_mcam_free_entry_req *req, 2573 struct msg_rsp *rsp) 2574 { 2575 struct npc_mcam *mcam = &rvu->hw->mcam; 2576 u16 pcifunc = req->hdr.pcifunc; 2577 int blkaddr, rc = 0; 2578 u16 cntr; 2579 2580 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2581 if (blkaddr < 0) 2582 return NPC_MCAM_INVALID_REQ; 2583 2584 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2585 if (!is_nixlf_attached(rvu, pcifunc)) 2586 return NPC_MCAM_INVALID_REQ; 2587 2588 mutex_lock(&mcam->lock); 2589 2590 if (req->all) 2591 goto free_all; 2592 2593 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2594 if (rc) 2595 goto exit; 2596 2597 mcam->entry2pfvf_map[req->entry] = 0; 2598 mcam->entry2target_pffunc[req->entry] = 0x0; 2599 npc_mcam_clear_bit(mcam, req->entry); 2600 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2601 2602 /* Update entry2counter mapping */ 2603 cntr = mcam->entry2cntr_map[req->entry]; 2604 if (cntr != NPC_MCAM_INVALID_MAP) 2605 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2606 req->entry, cntr); 2607 2608 goto exit; 2609 2610 free_all: 2611 /* Free up all entries allocated to requesting PFFUNC */ 2612 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2613 exit: 2614 mutex_unlock(&mcam->lock); 2615 return rc; 2616 } 2617 2618 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2619 struct npc_mcam_read_entry_req *req, 2620 struct npc_mcam_read_entry_rsp *rsp) 2621 { 2622 struct npc_mcam *mcam = &rvu->hw->mcam; 2623 u16 pcifunc = req->hdr.pcifunc; 2624 int blkaddr, rc; 2625 2626 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2627 if (blkaddr < 0) 2628 return NPC_MCAM_INVALID_REQ; 2629 2630 mutex_lock(&mcam->lock); 2631 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2632 if (!rc) { 2633 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2634 &rsp->entry_data, 2635 &rsp->intf, &rsp->enable); 2636 } 2637 2638 mutex_unlock(&mcam->lock); 2639 return rc; 2640 } 2641 2642 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2643 struct npc_mcam_write_entry_req *req, 2644 struct msg_rsp *rsp) 2645 { 2646 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2647 struct npc_mcam *mcam = &rvu->hw->mcam; 2648 u16 pcifunc = req->hdr.pcifunc; 2649 u16 channel, chan_mask; 2650 int blkaddr, rc; 2651 u8 nix_intf; 2652 2653 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2654 if (blkaddr < 0) 2655 return NPC_MCAM_INVALID_REQ; 2656 2657 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 2658 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 2659 channel &= chan_mask; 2660 2661 mutex_lock(&mcam->lock); 2662 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2663 if (rc) 2664 goto exit; 2665 2666 if (req->set_cntr && 2667 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2668 rc = NPC_MCAM_INVALID_REQ; 2669 goto exit; 2670 } 2671 2672 if (!is_npc_interface_valid(rvu, req->intf)) { 2673 rc = NPC_MCAM_INVALID_REQ; 2674 goto exit; 2675 } 2676 2677 if (is_npc_intf_tx(req->intf)) 2678 nix_intf = pfvf->nix_tx_intf; 2679 else 2680 nix_intf = pfvf->nix_rx_intf; 2681 2682 if (npc_mcam_verify_channel(rvu, pcifunc, req->intf, channel)) { 2683 rc = NPC_MCAM_INVALID_REQ; 2684 goto exit; 2685 } 2686 2687 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 2688 pcifunc)) { 2689 rc = NPC_MCAM_INVALID_REQ; 2690 goto exit; 2691 } 2692 2693 /* For AF installed rules, the nix_intf should be set to target NIX */ 2694 if (is_pffunc_af(req->hdr.pcifunc)) 2695 nix_intf = req->intf; 2696 2697 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2698 &req->entry_data, req->enable_entry); 2699 2700 if (req->set_cntr) 2701 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2702 req->entry, req->cntr); 2703 2704 rc = 0; 2705 exit: 2706 mutex_unlock(&mcam->lock); 2707 return rc; 2708 } 2709 2710 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2711 struct npc_mcam_ena_dis_entry_req *req, 2712 struct msg_rsp *rsp) 2713 { 2714 struct npc_mcam *mcam = &rvu->hw->mcam; 2715 u16 pcifunc = req->hdr.pcifunc; 2716 int blkaddr, rc; 2717 2718 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2719 if (blkaddr < 0) 2720 return NPC_MCAM_INVALID_REQ; 2721 2722 mutex_lock(&mcam->lock); 2723 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2724 mutex_unlock(&mcam->lock); 2725 if (rc) 2726 return rc; 2727 2728 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2729 2730 return 0; 2731 } 2732 2733 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2734 struct npc_mcam_ena_dis_entry_req *req, 2735 struct msg_rsp *rsp) 2736 { 2737 struct npc_mcam *mcam = &rvu->hw->mcam; 2738 u16 pcifunc = req->hdr.pcifunc; 2739 int blkaddr, rc; 2740 2741 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2742 if (blkaddr < 0) 2743 return NPC_MCAM_INVALID_REQ; 2744 2745 mutex_lock(&mcam->lock); 2746 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2747 mutex_unlock(&mcam->lock); 2748 if (rc) 2749 return rc; 2750 2751 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2752 2753 return 0; 2754 } 2755 2756 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2757 struct npc_mcam_shift_entry_req *req, 2758 struct npc_mcam_shift_entry_rsp *rsp) 2759 { 2760 struct npc_mcam *mcam = &rvu->hw->mcam; 2761 u16 pcifunc = req->hdr.pcifunc; 2762 u16 old_entry, new_entry; 2763 u16 index, cntr; 2764 int blkaddr, rc; 2765 2766 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2767 if (blkaddr < 0) 2768 return NPC_MCAM_INVALID_REQ; 2769 2770 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2771 return NPC_MCAM_INVALID_REQ; 2772 2773 mutex_lock(&mcam->lock); 2774 for (index = 0; index < req->shift_count; index++) { 2775 old_entry = req->curr_entry[index]; 2776 new_entry = req->new_entry[index]; 2777 2778 /* Check if both old and new entries are valid and 2779 * does belong to this PFFUNC or not. 2780 */ 2781 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2782 if (rc) 2783 break; 2784 2785 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2786 if (rc) 2787 break; 2788 2789 /* new_entry should not have a counter mapped */ 2790 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2791 rc = NPC_MCAM_PERM_DENIED; 2792 break; 2793 } 2794 2795 /* Disable the new_entry */ 2796 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2797 2798 /* Copy rule from old entry to new entry */ 2799 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2800 2801 /* Copy counter mapping, if any */ 2802 cntr = mcam->entry2cntr_map[old_entry]; 2803 if (cntr != NPC_MCAM_INVALID_MAP) { 2804 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2805 old_entry, cntr); 2806 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2807 new_entry, cntr); 2808 } 2809 2810 /* Enable new_entry and disable old_entry */ 2811 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2812 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2813 } 2814 2815 /* If shift has failed then report the failed index */ 2816 if (index != req->shift_count) { 2817 rc = NPC_MCAM_PERM_DENIED; 2818 rsp->failed_entry_idx = index; 2819 } 2820 2821 mutex_unlock(&mcam->lock); 2822 return rc; 2823 } 2824 2825 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2826 struct npc_mcam_alloc_counter_req *req, 2827 struct npc_mcam_alloc_counter_rsp *rsp) 2828 { 2829 struct npc_mcam *mcam = &rvu->hw->mcam; 2830 u16 pcifunc = req->hdr.pcifunc; 2831 u16 max_contig, cntr; 2832 int blkaddr, index; 2833 2834 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2835 if (blkaddr < 0) 2836 return NPC_MCAM_INVALID_REQ; 2837 2838 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2839 if (!is_nixlf_attached(rvu, pcifunc)) 2840 return NPC_MCAM_INVALID_REQ; 2841 2842 /* Since list of allocated counter IDs needs to be sent to requester, 2843 * max number of non-contiguous counters per mbox msg is limited. 2844 */ 2845 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2846 return NPC_MCAM_INVALID_REQ; 2847 2848 mutex_lock(&mcam->lock); 2849 2850 /* Check if unused counters are available or not */ 2851 if (!rvu_rsrc_free_count(&mcam->counters)) { 2852 mutex_unlock(&mcam->lock); 2853 return NPC_MCAM_ALLOC_FAILED; 2854 } 2855 2856 rsp->count = 0; 2857 2858 if (req->contig) { 2859 /* Allocate requested number of contiguous counters, if 2860 * unsuccessful find max contiguous entries available. 2861 */ 2862 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2863 mcam->counters.max, 0, 2864 req->count, &max_contig); 2865 rsp->count = max_contig; 2866 rsp->cntr = index; 2867 for (cntr = index; cntr < (index + max_contig); cntr++) { 2868 __set_bit(cntr, mcam->counters.bmap); 2869 mcam->cntr2pfvf_map[cntr] = pcifunc; 2870 } 2871 } else { 2872 /* Allocate requested number of non-contiguous counters, 2873 * if unsuccessful allocate as many as possible. 2874 */ 2875 for (cntr = 0; cntr < req->count; cntr++) { 2876 index = rvu_alloc_rsrc(&mcam->counters); 2877 if (index < 0) 2878 break; 2879 rsp->cntr_list[cntr] = index; 2880 rsp->count++; 2881 mcam->cntr2pfvf_map[index] = pcifunc; 2882 } 2883 } 2884 2885 mutex_unlock(&mcam->lock); 2886 return 0; 2887 } 2888 2889 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2890 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2891 { 2892 struct npc_mcam *mcam = &rvu->hw->mcam; 2893 u16 index, entry = 0; 2894 int blkaddr, err; 2895 2896 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2897 if (blkaddr < 0) 2898 return NPC_MCAM_INVALID_REQ; 2899 2900 mutex_lock(&mcam->lock); 2901 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2902 if (err) { 2903 mutex_unlock(&mcam->lock); 2904 return err; 2905 } 2906 2907 /* Mark counter as free/unused */ 2908 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2909 rvu_free_rsrc(&mcam->counters, req->cntr); 2910 2911 /* Disable all MCAM entry's stats which are using this counter */ 2912 while (entry < mcam->bmap_entries) { 2913 if (!mcam->cntr_refcnt[req->cntr]) 2914 break; 2915 2916 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2917 if (index >= mcam->bmap_entries) 2918 break; 2919 entry = index + 1; 2920 if (mcam->entry2cntr_map[index] != req->cntr) 2921 continue; 2922 2923 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2924 index, req->cntr); 2925 } 2926 2927 mutex_unlock(&mcam->lock); 2928 return 0; 2929 } 2930 2931 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2932 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2933 { 2934 struct npc_mcam *mcam = &rvu->hw->mcam; 2935 u16 index, entry = 0; 2936 int blkaddr, rc; 2937 2938 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2939 if (blkaddr < 0) 2940 return NPC_MCAM_INVALID_REQ; 2941 2942 mutex_lock(&mcam->lock); 2943 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2944 if (rc) 2945 goto exit; 2946 2947 /* Unmap the MCAM entry and counter */ 2948 if (!req->all) { 2949 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2950 if (rc) 2951 goto exit; 2952 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2953 req->entry, req->cntr); 2954 goto exit; 2955 } 2956 2957 /* Disable all MCAM entry's stats which are using this counter */ 2958 while (entry < mcam->bmap_entries) { 2959 if (!mcam->cntr_refcnt[req->cntr]) 2960 break; 2961 2962 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2963 if (index >= mcam->bmap_entries) 2964 break; 2965 if (mcam->entry2cntr_map[index] != req->cntr) 2966 continue; 2967 2968 entry = index + 1; 2969 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2970 index, req->cntr); 2971 } 2972 exit: 2973 mutex_unlock(&mcam->lock); 2974 return rc; 2975 } 2976 2977 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2978 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2979 { 2980 struct npc_mcam *mcam = &rvu->hw->mcam; 2981 int blkaddr, err; 2982 2983 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2984 if (blkaddr < 0) 2985 return NPC_MCAM_INVALID_REQ; 2986 2987 mutex_lock(&mcam->lock); 2988 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2989 mutex_unlock(&mcam->lock); 2990 if (err) 2991 return err; 2992 2993 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 2994 2995 return 0; 2996 } 2997 2998 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 2999 struct npc_mcam_oper_counter_req *req, 3000 struct npc_mcam_oper_counter_rsp *rsp) 3001 { 3002 struct npc_mcam *mcam = &rvu->hw->mcam; 3003 int blkaddr, err; 3004 3005 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3006 if (blkaddr < 0) 3007 return NPC_MCAM_INVALID_REQ; 3008 3009 mutex_lock(&mcam->lock); 3010 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 3011 mutex_unlock(&mcam->lock); 3012 if (err) 3013 return err; 3014 3015 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 3016 rsp->stat &= BIT_ULL(48) - 1; 3017 3018 return 0; 3019 } 3020 3021 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 3022 struct npc_mcam_alloc_and_write_entry_req *req, 3023 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 3024 { 3025 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 3026 struct npc_mcam_alloc_counter_req cntr_req; 3027 struct npc_mcam_alloc_counter_rsp cntr_rsp; 3028 struct npc_mcam_alloc_entry_req entry_req; 3029 struct npc_mcam_alloc_entry_rsp entry_rsp; 3030 struct npc_mcam *mcam = &rvu->hw->mcam; 3031 u16 entry = NPC_MCAM_ENTRY_INVALID; 3032 u16 cntr = NPC_MCAM_ENTRY_INVALID; 3033 u16 channel, chan_mask; 3034 int blkaddr, rc; 3035 u8 nix_intf; 3036 3037 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3038 if (blkaddr < 0) 3039 return NPC_MCAM_INVALID_REQ; 3040 3041 if (!is_npc_interface_valid(rvu, req->intf)) 3042 return NPC_MCAM_INVALID_REQ; 3043 3044 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 3045 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 3046 channel &= chan_mask; 3047 3048 if (npc_mcam_verify_channel(rvu, req->hdr.pcifunc, req->intf, channel)) 3049 return NPC_MCAM_INVALID_REQ; 3050 3051 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 3052 req->hdr.pcifunc)) 3053 return NPC_MCAM_INVALID_REQ; 3054 3055 /* Try to allocate a MCAM entry */ 3056 entry_req.hdr.pcifunc = req->hdr.pcifunc; 3057 entry_req.contig = true; 3058 entry_req.priority = req->priority; 3059 entry_req.ref_entry = req->ref_entry; 3060 entry_req.count = 1; 3061 3062 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 3063 &entry_req, &entry_rsp); 3064 if (rc) 3065 return rc; 3066 3067 if (!entry_rsp.count) 3068 return NPC_MCAM_ALLOC_FAILED; 3069 3070 entry = entry_rsp.entry; 3071 3072 if (!req->alloc_cntr) 3073 goto write_entry; 3074 3075 /* Now allocate counter */ 3076 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 3077 cntr_req.contig = true; 3078 cntr_req.count = 1; 3079 3080 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 3081 if (rc) { 3082 /* Free allocated MCAM entry */ 3083 mutex_lock(&mcam->lock); 3084 mcam->entry2pfvf_map[entry] = 0; 3085 npc_mcam_clear_bit(mcam, entry); 3086 mutex_unlock(&mcam->lock); 3087 return rc; 3088 } 3089 3090 cntr = cntr_rsp.cntr; 3091 3092 write_entry: 3093 mutex_lock(&mcam->lock); 3094 3095 if (is_npc_intf_tx(req->intf)) 3096 nix_intf = pfvf->nix_tx_intf; 3097 else 3098 nix_intf = pfvf->nix_rx_intf; 3099 3100 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 3101 &req->entry_data, req->enable_entry); 3102 3103 if (req->alloc_cntr) 3104 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 3105 mutex_unlock(&mcam->lock); 3106 3107 rsp->entry = entry; 3108 rsp->cntr = cntr; 3109 3110 return 0; 3111 } 3112 3113 #define GET_KEX_CFG(intf) \ 3114 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 3115 3116 #define GET_KEX_FLAGS(ld) \ 3117 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 3118 3119 #define GET_KEX_LD(intf, lid, lt, ld) \ 3120 rvu_read64(rvu, BLKADDR_NPC, \ 3121 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 3122 3123 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 3124 rvu_read64(rvu, BLKADDR_NPC, \ 3125 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 3126 3127 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 3128 struct npc_get_kex_cfg_rsp *rsp) 3129 { 3130 int lid, lt, ld, fl; 3131 3132 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 3133 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 3134 for (lid = 0; lid < NPC_MAX_LID; lid++) { 3135 for (lt = 0; lt < NPC_MAX_LT; lt++) { 3136 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3137 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 3138 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 3139 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 3140 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 3141 } 3142 } 3143 } 3144 for (ld = 0; ld < NPC_MAX_LD; ld++) 3145 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 3146 3147 for (ld = 0; ld < NPC_MAX_LD; ld++) { 3148 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 3149 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 3150 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 3151 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 3152 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 3153 } 3154 } 3155 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 3156 return 0; 3157 } 3158 3159 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 3160 struct msg_req *req, 3161 struct npc_mcam_read_base_rule_rsp *rsp) 3162 { 3163 struct npc_mcam *mcam = &rvu->hw->mcam; 3164 int index, blkaddr, nixlf, rc = 0; 3165 u16 pcifunc = req->hdr.pcifunc; 3166 struct rvu_pfvf *pfvf; 3167 u8 intf, enable; 3168 3169 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3170 if (blkaddr < 0) 3171 return NPC_MCAM_INVALID_REQ; 3172 3173 /* Return the channel number in case of PF */ 3174 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 3175 pfvf = rvu_get_pfvf(rvu, pcifunc); 3176 rsp->entry.kw[0] = pfvf->rx_chan_base; 3177 rsp->entry.kw_mask[0] = 0xFFFULL; 3178 goto out; 3179 } 3180 3181 /* Find the pkt steering rule installed by PF to this VF */ 3182 mutex_lock(&mcam->lock); 3183 for (index = 0; index < mcam->bmap_entries; index++) { 3184 if (mcam->entry2target_pffunc[index] == pcifunc) 3185 goto read_entry; 3186 } 3187 3188 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3189 if (rc < 0) { 3190 mutex_unlock(&mcam->lock); 3191 goto out; 3192 } 3193 /* Read the default ucast entry if there is no pkt steering rule */ 3194 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 3195 NIXLF_UCAST_ENTRY); 3196 read_entry: 3197 /* Read the mcam entry */ 3198 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 3199 &enable); 3200 mutex_unlock(&mcam->lock); 3201 out: 3202 return rc; 3203 } 3204 3205 int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, 3206 struct npc_mcam_get_stats_req *req, 3207 struct npc_mcam_get_stats_rsp *rsp) 3208 { 3209 struct npc_mcam *mcam = &rvu->hw->mcam; 3210 u16 index, cntr; 3211 int blkaddr; 3212 u64 regval; 3213 u32 bank; 3214 3215 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 3216 if (blkaddr < 0) 3217 return NPC_MCAM_INVALID_REQ; 3218 3219 mutex_lock(&mcam->lock); 3220 3221 index = req->entry & (mcam->banksize - 1); 3222 bank = npc_get_bank(mcam, req->entry); 3223 3224 /* read MCAM entry STAT_ACT register */ 3225 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); 3226 3227 if (!(regval & BIT_ULL(9))) { 3228 rsp->stat_ena = 0; 3229 mutex_unlock(&mcam->lock); 3230 return 0; 3231 } 3232 3233 cntr = regval & 0x1FF; 3234 3235 rsp->stat_ena = 1; 3236 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr)); 3237 rsp->stat &= BIT_ULL(48) - 1; 3238 3239 mutex_unlock(&mcam->lock); 3240 3241 return 0; 3242 } 3243