1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "cgx.h" 20 #include "npc_profile.h" 21 22 #define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ 23 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 24 25 #define NIXLF_UCAST_ENTRY 0 26 #define NIXLF_BCAST_ENTRY 1 27 #define NIXLF_PROMISC_ENTRY 2 28 29 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 30 #define NPC_HW_TSTAMP_OFFSET 8 31 #define NPC_KEX_CHAN_MASK 0xFFFULL 32 #define NPC_KEX_PF_FUNC_MASK 0xFFFFULL 33 34 static const char def_pfl_name[] = "default"; 35 36 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 37 int blkaddr, u16 pcifunc); 38 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 39 u16 pcifunc); 40 41 bool is_npc_intf_tx(u8 intf) 42 { 43 return !!(intf & 0x1); 44 } 45 46 bool is_npc_intf_rx(u8 intf) 47 { 48 return !(intf & 0x1); 49 } 50 51 bool is_npc_interface_valid(struct rvu *rvu, u8 intf) 52 { 53 struct rvu_hwinfo *hw = rvu->hw; 54 55 return intf < hw->npc_intfs; 56 } 57 58 int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena) 59 { 60 /* Due to a HW issue in these silicon versions, parse nibble enable 61 * configuration has to be identical for both Rx and Tx interfaces. 62 */ 63 if (is_rvu_96xx_B0(rvu)) 64 return nibble_ena; 65 return 0; 66 } 67 68 static int npc_mcam_verify_pf_func(struct rvu *rvu, 69 struct mcam_entry *entry_data, u8 intf, 70 u16 pcifunc) 71 { 72 u16 pf_func, pf_func_mask; 73 74 if (is_npc_intf_rx(intf)) 75 return 0; 76 77 pf_func_mask = (entry_data->kw_mask[0] >> 32) & 78 NPC_KEX_PF_FUNC_MASK; 79 pf_func = (entry_data->kw[0] >> 32) & NPC_KEX_PF_FUNC_MASK; 80 81 pf_func = be16_to_cpu((__force __be16)pf_func); 82 if (pf_func_mask != NPC_KEX_PF_FUNC_MASK || 83 ((pf_func & ~RVU_PFVF_FUNC_MASK) != 84 (pcifunc & ~RVU_PFVF_FUNC_MASK))) 85 return -EINVAL; 86 87 return 0; 88 } 89 90 int npc_mcam_verify_channel(struct rvu *rvu, u16 pcifunc, u8 intf, u16 channel) 91 { 92 int pf = rvu_get_pf(pcifunc); 93 u8 cgx_id, lmac_id; 94 int base = 0, end; 95 96 if (is_npc_intf_tx(intf)) 97 return 0; 98 99 if (is_afvf(pcifunc)) { 100 end = rvu_get_num_lbk_chans(); 101 if (end < 0) 102 return -EINVAL; 103 } else { 104 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 105 base = NIX_CHAN_CGX_LMAC_CHX(cgx_id, lmac_id, 0x0); 106 /* CGX mapped functions has maximum of 16 channels */ 107 end = NIX_CHAN_CGX_LMAC_CHX(cgx_id, lmac_id, 0xF); 108 } 109 110 if (channel < base || channel > end) 111 return -EINVAL; 112 113 return 0; 114 } 115 116 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 117 { 118 int blkaddr; 119 u64 val = 0; 120 121 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 122 if (blkaddr < 0) 123 return; 124 125 /* Config CPI base for the PKIND */ 126 val = pkind | 1ULL << 62; 127 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 128 } 129 130 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 131 { 132 struct npc_pkind *pkind = &rvu->hw->pkind; 133 u32 map; 134 int i; 135 136 for (i = 0; i < pkind->rsrc.max; i++) { 137 map = pkind->pfchan_map[i]; 138 if (((map >> 16) & 0x3F) == pf) 139 return i; 140 } 141 return -1; 142 } 143 144 #define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20) 145 146 int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable) 147 { 148 int pkind, blkaddr; 149 u64 val; 150 151 pkind = rvu_npc_get_pkind(rvu, pf); 152 if (pkind < 0) { 153 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__); 154 return -EINVAL; 155 } 156 157 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc); 158 if (blkaddr < 0) { 159 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 160 return -EINVAL; 161 } 162 163 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind)); 164 val &= ~NPC_AF_ACTION0_PTR_ADVANCE; 165 /* If timestamp is enabled then configure NPC to shift 8 bytes */ 166 if (enable) 167 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE, 168 NPC_HW_TSTAMP_OFFSET); 169 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val); 170 171 return 0; 172 } 173 174 static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc, 175 int nixlf) 176 { 177 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); 178 struct rvu *rvu = hw->rvu; 179 int blkaddr = 0, max = 0; 180 struct rvu_block *block; 181 struct rvu_pfvf *pfvf; 182 183 pfvf = rvu_get_pfvf(rvu, pcifunc); 184 /* Given a PF/VF and NIX LF number calculate the unicast mcam 185 * entry index based on the NIX block assigned to the PF/VF. 186 */ 187 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 188 while (blkaddr) { 189 if (pfvf->nix_blkaddr == blkaddr) 190 break; 191 block = &rvu->hw->block[blkaddr]; 192 max += block->lf.max; 193 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 194 } 195 196 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF; 197 } 198 199 static int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 200 u16 pcifunc, int nixlf, int type) 201 { 202 int pf = rvu_get_pf(pcifunc); 203 int index; 204 205 /* Check if this is for a PF */ 206 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 207 /* Reserved entries exclude PF0 */ 208 pf--; 209 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 210 /* Broadcast address matching entry should be first so 211 * that the packet can be replicated to all VFs. 212 */ 213 if (type == NIXLF_BCAST_ENTRY) 214 return index; 215 else if (type == NIXLF_PROMISC_ENTRY) 216 return index + 1; 217 } 218 219 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf); 220 } 221 222 int npc_get_bank(struct npc_mcam *mcam, int index) 223 { 224 int bank = index / mcam->banksize; 225 226 /* 0,1 & 2,3 banks are combined for this keysize */ 227 if (mcam->keysize == NPC_MCAM_KEY_X2) 228 return bank ? 2 : 0; 229 230 return bank; 231 } 232 233 static bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 234 int blkaddr, int index) 235 { 236 int bank = npc_get_bank(mcam, index); 237 u64 cfg; 238 239 index &= (mcam->banksize - 1); 240 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 241 return (cfg & 1); 242 } 243 244 void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 245 int blkaddr, int index, bool enable) 246 { 247 int bank = npc_get_bank(mcam, index); 248 int actbank = bank; 249 250 index &= (mcam->banksize - 1); 251 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 252 rvu_write64(rvu, blkaddr, 253 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 254 enable ? 1 : 0); 255 } 256 } 257 258 static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 259 int blkaddr, int index) 260 { 261 int bank = npc_get_bank(mcam, index); 262 int actbank = bank; 263 264 index &= (mcam->banksize - 1); 265 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 266 rvu_write64(rvu, blkaddr, 267 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0); 268 rvu_write64(rvu, blkaddr, 269 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0); 270 271 rvu_write64(rvu, blkaddr, 272 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0); 273 rvu_write64(rvu, blkaddr, 274 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0); 275 276 rvu_write64(rvu, blkaddr, 277 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0); 278 rvu_write64(rvu, blkaddr, 279 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0); 280 } 281 } 282 283 static void npc_get_keyword(struct mcam_entry *entry, int idx, 284 u64 *cam0, u64 *cam1) 285 { 286 u64 kw_mask = 0x00; 287 288 #define CAM_MASK(n) (BIT_ULL(n) - 1) 289 290 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 291 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 292 * 293 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 294 */ 295 switch (idx) { 296 case 0: 297 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 298 *cam1 = entry->kw[0]; 299 kw_mask = entry->kw_mask[0]; 300 break; 301 case 1: 302 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 303 *cam1 = entry->kw[1] & CAM_MASK(48); 304 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 305 break; 306 case 2: 307 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 308 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 309 */ 310 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 311 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 312 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 313 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 314 break; 315 case 3: 316 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 317 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 318 */ 319 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 320 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 321 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 322 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 323 break; 324 case 4: 325 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 326 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 327 */ 328 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 329 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 330 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 331 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 332 break; 333 case 5: 334 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 335 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 336 */ 337 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 338 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 339 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 340 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 341 break; 342 case 6: 343 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 344 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 345 */ 346 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 347 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 348 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 349 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 350 break; 351 case 7: 352 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 353 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 354 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 355 break; 356 } 357 358 *cam1 &= kw_mask; 359 *cam0 = ~*cam1 & kw_mask; 360 } 361 362 static void npc_fill_entryword(struct mcam_entry *entry, int idx, 363 u64 cam0, u64 cam1) 364 { 365 /* Similar to npc_get_keyword, but fills mcam_entry structure from 366 * CAM registers. 367 */ 368 switch (idx) { 369 case 0: 370 entry->kw[0] = cam1; 371 entry->kw_mask[0] = cam1 ^ cam0; 372 break; 373 case 1: 374 entry->kw[1] = cam1; 375 entry->kw_mask[1] = cam1 ^ cam0; 376 break; 377 case 2: 378 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48; 379 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48); 380 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 381 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48); 382 break; 383 case 3: 384 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48; 385 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32); 386 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48; 387 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32); 388 break; 389 case 4: 390 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32; 391 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32); 392 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 393 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32); 394 break; 395 case 5: 396 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32; 397 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16); 398 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32; 399 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16); 400 break; 401 case 6: 402 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16; 403 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16); 404 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 405 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16); 406 break; 407 case 7: 408 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16; 409 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16; 410 break; 411 } 412 } 413 414 static void npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, 415 int blkaddr, int index, 416 struct mcam_entry *entry) 417 { 418 u16 owner, target_func; 419 struct rvu_pfvf *pfvf; 420 int bank, nixlf; 421 u64 rx_action; 422 423 owner = mcam->entry2pfvf_map[index]; 424 target_func = (entry->action >> 4) & 0xffff; 425 /* return incase target is PF or LBK or rule owner is not PF */ 426 if (is_afvf(target_func) || (owner & RVU_PFVF_FUNC_MASK) || 427 !(target_func & RVU_PFVF_FUNC_MASK)) 428 return; 429 430 pfvf = rvu_get_pfvf(rvu, target_func); 431 mcam->entry2target_pffunc[index] = target_func; 432 /* return if nixlf is not attached or initialized */ 433 if (!is_nixlf_attached(rvu, target_func) || !pfvf->def_ucast_rule) 434 return; 435 436 /* get VF ucast entry rule */ 437 nix_get_nixlf(rvu, target_func, &nixlf, NULL); 438 index = npc_get_nixlf_mcam_index(mcam, target_func, 439 nixlf, NIXLF_UCAST_ENTRY); 440 bank = npc_get_bank(mcam, index); 441 index &= (mcam->banksize - 1); 442 443 rx_action = rvu_read64(rvu, blkaddr, 444 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 445 if (rx_action) 446 entry->action = rx_action; 447 } 448 449 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 450 int blkaddr, int index, u8 intf, 451 struct mcam_entry *entry, bool enable) 452 { 453 int bank = npc_get_bank(mcam, index); 454 int kw = 0, actbank, actindex; 455 u64 cam0, cam1; 456 457 actbank = bank; /* Save bank id, to set action later on */ 458 actindex = index; 459 index &= (mcam->banksize - 1); 460 461 /* Disable before mcam entry update */ 462 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 463 464 /* Clear mcam entry to avoid writes being suppressed by NPC */ 465 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex); 466 467 /* CAM1 takes the comparison value and 468 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 469 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 470 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 471 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 472 */ 473 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 474 /* Interface should be set in all banks */ 475 rvu_write64(rvu, blkaddr, 476 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 477 intf); 478 rvu_write64(rvu, blkaddr, 479 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 480 ~intf & 0x3); 481 482 /* Set the match key */ 483 npc_get_keyword(entry, kw, &cam0, &cam1); 484 rvu_write64(rvu, blkaddr, 485 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 486 rvu_write64(rvu, blkaddr, 487 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 488 489 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 490 rvu_write64(rvu, blkaddr, 491 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 492 rvu_write64(rvu, blkaddr, 493 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 494 } 495 496 /* copy VF default entry action to the VF mcam entry */ 497 if (intf == NIX_INTF_RX && actindex < mcam->bmap_entries) 498 npc_get_default_entry_action(rvu, mcam, blkaddr, actindex, 499 entry); 500 501 /* Set 'action' */ 502 rvu_write64(rvu, blkaddr, 503 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 504 505 /* Set TAG 'action' */ 506 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 507 entry->vtag_action); 508 509 /* Enable the entry */ 510 if (enable) 511 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 512 } 513 514 void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 515 int blkaddr, u16 src, 516 struct mcam_entry *entry, u8 *intf, u8 *ena) 517 { 518 int sbank = npc_get_bank(mcam, src); 519 int bank, kw = 0; 520 u64 cam0, cam1; 521 522 src &= (mcam->banksize - 1); 523 bank = sbank; 524 525 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 526 cam1 = rvu_read64(rvu, blkaddr, 527 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1)); 528 cam0 = rvu_read64(rvu, blkaddr, 529 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0)); 530 npc_fill_entryword(entry, kw, cam0, cam1); 531 532 cam1 = rvu_read64(rvu, blkaddr, 533 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1)); 534 cam0 = rvu_read64(rvu, blkaddr, 535 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0)); 536 npc_fill_entryword(entry, kw + 1, cam0, cam1); 537 } 538 539 entry->action = rvu_read64(rvu, blkaddr, 540 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 541 entry->vtag_action = 542 rvu_read64(rvu, blkaddr, 543 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 544 *intf = rvu_read64(rvu, blkaddr, 545 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3; 546 *ena = rvu_read64(rvu, blkaddr, 547 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1; 548 } 549 550 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 551 int blkaddr, u16 src, u16 dest) 552 { 553 int dbank = npc_get_bank(mcam, dest); 554 int sbank = npc_get_bank(mcam, src); 555 u64 cfg, sreg, dreg; 556 int bank, i; 557 558 src &= (mcam->banksize - 1); 559 dest &= (mcam->banksize - 1); 560 561 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 562 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 563 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 564 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 565 for (i = 0; i < 6; i++) { 566 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 567 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 568 } 569 } 570 571 /* Copy action */ 572 cfg = rvu_read64(rvu, blkaddr, 573 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 574 rvu_write64(rvu, blkaddr, 575 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 576 577 /* Copy TAG action */ 578 cfg = rvu_read64(rvu, blkaddr, 579 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 580 rvu_write64(rvu, blkaddr, 581 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 582 583 /* Enable or disable */ 584 cfg = rvu_read64(rvu, blkaddr, 585 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 586 rvu_write64(rvu, blkaddr, 587 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 588 } 589 590 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 591 int blkaddr, int index) 592 { 593 int bank = npc_get_bank(mcam, index); 594 595 index &= (mcam->banksize - 1); 596 return rvu_read64(rvu, blkaddr, 597 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 598 } 599 600 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 601 int nixlf, u64 chan, u8 *mac_addr) 602 { 603 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 604 struct npc_install_flow_req req = { 0 }; 605 struct npc_install_flow_rsp rsp = { 0 }; 606 struct npc_mcam *mcam = &rvu->hw->mcam; 607 struct nix_rx_action action; 608 int blkaddr, index; 609 610 /* AF's VFs work in promiscuous mode */ 611 if (is_afvf(pcifunc)) 612 return; 613 614 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 615 if (blkaddr < 0) 616 return; 617 618 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 619 nixlf, NIXLF_UCAST_ENTRY); 620 621 /* Don't change the action if entry is already enabled 622 * Otherwise RSS action may get overwritten. 623 */ 624 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 625 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 626 blkaddr, index); 627 } else { 628 *(u64 *)&action = 0x00; 629 action.op = NIX_RX_ACTIONOP_UCAST; 630 action.pf_func = pcifunc; 631 } 632 633 req.default_rule = 1; 634 ether_addr_copy(req.packet.dmac, mac_addr); 635 eth_broadcast_addr((u8 *)&req.mask.dmac); 636 req.features = BIT_ULL(NPC_DMAC); 637 req.channel = chan; 638 req.intf = pfvf->nix_rx_intf; 639 req.op = action.op; 640 req.hdr.pcifunc = 0; /* AF is requester */ 641 req.vf = action.pf_func; 642 req.index = action.index; 643 req.match_id = action.match_id; 644 req.flow_key_alg = action.flow_key_alg; 645 646 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); 647 } 648 649 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 650 int nixlf, u64 chan, bool allmulti) 651 { 652 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 653 struct npc_mcam *mcam = &rvu->hw->mcam; 654 int blkaddr, ucast_idx, index, kwi; 655 struct mcam_entry entry = { {0} }; 656 struct nix_rx_action action = { }; 657 658 /* Only PF or AF VF can add a promiscuous entry */ 659 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) 660 return; 661 662 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 663 if (blkaddr < 0) 664 return; 665 666 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 667 nixlf, NIXLF_PROMISC_ENTRY); 668 669 entry.kw[0] = chan; 670 entry.kw_mask[0] = 0xFFFULL; 671 672 if (allmulti) { 673 kwi = NPC_KEXOF_DMAC / sizeof(u64); 674 entry.kw[kwi] = BIT_ULL(40); /* LSB bit of 1st byte in DMAC */ 675 entry.kw_mask[kwi] = BIT_ULL(40); 676 } 677 678 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 679 nixlf, NIXLF_UCAST_ENTRY); 680 681 /* If the corresponding PF's ucast action is RSS, 682 * use the same action for promisc also 683 */ 684 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 685 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 686 blkaddr, ucast_idx); 687 688 if (action.op != NIX_RX_ACTIONOP_RSS) { 689 *(u64 *)&action = 0x00; 690 action.op = NIX_RX_ACTIONOP_UCAST; 691 action.pf_func = pcifunc; 692 } 693 694 entry.action = *(u64 *)&action; 695 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 696 pfvf->nix_rx_intf, &entry, true); 697 } 698 699 static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, 700 int nixlf, bool enable) 701 { 702 struct npc_mcam *mcam = &rvu->hw->mcam; 703 int blkaddr, index; 704 705 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 706 if (blkaddr < 0) 707 return; 708 709 /* Only PF's have a promiscuous entry */ 710 if (pcifunc & RVU_PFVF_FUNC_MASK) 711 return; 712 713 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 714 nixlf, NIXLF_PROMISC_ENTRY); 715 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 716 } 717 718 void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 719 { 720 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); 721 } 722 723 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 724 { 725 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); 726 } 727 728 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 729 int nixlf, u64 chan) 730 { 731 struct npc_mcam *mcam = &rvu->hw->mcam; 732 struct mcam_entry entry = { {0} }; 733 struct rvu_hwinfo *hw = rvu->hw; 734 struct nix_rx_action action; 735 struct rvu_pfvf *pfvf; 736 int blkaddr, index; 737 738 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 739 if (blkaddr < 0) 740 return; 741 742 /* Skip LBK VFs */ 743 if (is_afvf(pcifunc)) 744 return; 745 746 /* If pkt replication is not supported, 747 * then only PF is allowed to add a bcast match entry. 748 */ 749 if (!hw->cap.nix_rx_multicast && pcifunc & RVU_PFVF_FUNC_MASK) 750 return; 751 752 /* Get 'pcifunc' of PF device */ 753 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 754 pfvf = rvu_get_pfvf(rvu, pcifunc); 755 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 756 nixlf, NIXLF_BCAST_ENTRY); 757 758 /* Match ingress channel */ 759 entry.kw[0] = chan; 760 entry.kw_mask[0] = 0xfffull; 761 762 /* Match broadcast MAC address. 763 * DMAC is extracted at 0th bit of PARSE_KEX::KW1 764 */ 765 entry.kw[1] = 0xffffffffffffull; 766 entry.kw_mask[1] = 0xffffffffffffull; 767 768 *(u64 *)&action = 0x00; 769 if (!hw->cap.nix_rx_multicast) { 770 /* Early silicon doesn't support pkt replication, 771 * so install entry with UCAST action, so that PF 772 * receives all broadcast packets. 773 */ 774 action.op = NIX_RX_ACTIONOP_UCAST; 775 action.pf_func = pcifunc; 776 } else { 777 action.index = pfvf->bcast_mce_idx; 778 action.op = NIX_RX_ACTIONOP_MCAST; 779 } 780 781 entry.action = *(u64 *)&action; 782 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 783 pfvf->nix_rx_intf, &entry, true); 784 } 785 786 void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, bool enable) 787 { 788 struct npc_mcam *mcam = &rvu->hw->mcam; 789 int blkaddr, index; 790 791 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 792 if (blkaddr < 0) 793 return; 794 795 /* Get 'pcifunc' of PF device */ 796 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK; 797 798 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 0, NIXLF_BCAST_ENTRY); 799 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 800 } 801 802 static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, 803 int blkaddr, u16 pcifunc, u64 rx_action) 804 { 805 int actindex, index, bank; 806 bool enable; 807 808 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 809 return; 810 811 mutex_lock(&mcam->lock); 812 for (index = 0; index < mcam->bmap_entries; index++) { 813 if (mcam->entry2target_pffunc[index] == pcifunc) { 814 bank = npc_get_bank(mcam, index); 815 actindex = index; 816 index &= (mcam->banksize - 1); 817 818 /* read vf flow entry enable status */ 819 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, 820 actindex); 821 /* disable before mcam entry update */ 822 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, 823 false); 824 /* update 'action' */ 825 rvu_write64(rvu, blkaddr, 826 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 827 rx_action); 828 if (enable) 829 npc_enable_mcam_entry(rvu, mcam, blkaddr, 830 actindex, true); 831 } 832 } 833 mutex_unlock(&mcam->lock); 834 } 835 836 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 837 int group, int alg_idx, int mcam_index) 838 { 839 struct npc_mcam *mcam = &rvu->hw->mcam; 840 struct nix_rx_action action; 841 int blkaddr, index, bank; 842 struct rvu_pfvf *pfvf; 843 844 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 845 if (blkaddr < 0) 846 return; 847 848 /* Check if this is for reserved default entry */ 849 if (mcam_index < 0) { 850 if (group != DEFAULT_RSS_CONTEXT_GROUP) 851 return; 852 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 853 nixlf, NIXLF_UCAST_ENTRY); 854 } else { 855 /* TODO: validate this mcam index */ 856 index = mcam_index; 857 } 858 859 if (index >= mcam->total_entries) 860 return; 861 862 bank = npc_get_bank(mcam, index); 863 index &= (mcam->banksize - 1); 864 865 *(u64 *)&action = rvu_read64(rvu, blkaddr, 866 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 867 /* Ignore if no action was set earlier */ 868 if (!*(u64 *)&action) 869 return; 870 871 action.op = NIX_RX_ACTIONOP_RSS; 872 action.pf_func = pcifunc; 873 action.index = group; 874 action.flow_key_alg = alg_idx; 875 876 rvu_write64(rvu, blkaddr, 877 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 878 879 /* update the VF flow rule action with the VF default entry action */ 880 if (mcam_index < 0) 881 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, 882 *(u64 *)&action); 883 884 /* update the action change in default rule */ 885 pfvf = rvu_get_pfvf(rvu, pcifunc); 886 if (pfvf->def_ucast_rule) 887 pfvf->def_ucast_rule->rx_action = action; 888 889 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 890 nixlf, NIXLF_PROMISC_ENTRY); 891 892 /* If PF's promiscuous entry is enabled, 893 * Set RSS action for that entry as well 894 */ 895 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 896 bank = npc_get_bank(mcam, index); 897 index &= (mcam->banksize - 1); 898 899 rvu_write64(rvu, blkaddr, 900 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 901 *(u64 *)&action); 902 } 903 } 904 905 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 906 int nixlf, bool enable) 907 { 908 struct npc_mcam *mcam = &rvu->hw->mcam; 909 struct nix_rx_action action; 910 int index, bank, blkaddr; 911 912 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 913 if (blkaddr < 0) 914 return; 915 916 /* Ucast MCAM match entry of this PF/VF */ 917 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 918 nixlf, NIXLF_UCAST_ENTRY); 919 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 920 921 /* For PF, ena/dis promisc and bcast MCAM match entries. 922 * For VFs add/delete from bcast list when RX multicast 923 * feature is present. 924 */ 925 if (pcifunc & RVU_PFVF_FUNC_MASK && !rvu->hw->cap.nix_rx_multicast) 926 return; 927 928 /* For bcast, enable/disable only if it's action is not 929 * packet replication, incase if action is replication 930 * then this PF/VF's nixlf is removed from bcast replication 931 * list. 932 */ 933 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK, 934 nixlf, NIXLF_BCAST_ENTRY); 935 bank = npc_get_bank(mcam, index); 936 *(u64 *)&action = rvu_read64(rvu, blkaddr, 937 NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); 938 939 /* VFs will not have BCAST entry */ 940 if (action.op != NIX_RX_ACTIONOP_MCAST && 941 !(pcifunc & RVU_PFVF_FUNC_MASK)) { 942 npc_enable_mcam_entry(rvu, mcam, 943 blkaddr, index, enable); 944 } else { 945 nix_update_bcast_mce_list(rvu, pcifunc, enable); 946 /* Enable PF's BCAST entry for packet replication */ 947 rvu_npc_enable_bcast_entry(rvu, pcifunc, enable); 948 } 949 950 if (enable) 951 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); 952 else 953 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); 954 } 955 956 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 957 { 958 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 959 } 960 961 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 962 { 963 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 964 } 965 966 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 967 { 968 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 969 struct npc_mcam *mcam = &rvu->hw->mcam; 970 struct rvu_npc_mcam_rule *rule; 971 int blkaddr; 972 973 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 974 if (blkaddr < 0) 975 return; 976 977 mutex_lock(&mcam->lock); 978 979 /* Disable MCAM entries directing traffic to this 'pcifunc' */ 980 list_for_each_entry(rule, &mcam->mcam_rules, list) { 981 if (is_npc_intf_rx(rule->intf) && 982 rule->rx_action.pf_func == pcifunc) { 983 npc_enable_mcam_entry(rvu, mcam, blkaddr, 984 rule->entry, false); 985 rule->enable = false; 986 /* Indicate that default rule is disabled */ 987 if (rule->default_rule) 988 pfvf->def_ucast_rule = NULL; 989 } 990 } 991 992 mutex_unlock(&mcam->lock); 993 994 npc_mcam_disable_flows(rvu, pcifunc); 995 996 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 997 } 998 999 void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 1000 { 1001 struct npc_mcam *mcam = &rvu->hw->mcam; 1002 struct rvu_npc_mcam_rule *rule, *tmp; 1003 int blkaddr; 1004 1005 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1006 if (blkaddr < 0) 1007 return; 1008 1009 mutex_lock(&mcam->lock); 1010 1011 /* Free all MCAM entries owned by this 'pcifunc' */ 1012 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1013 1014 /* Free all MCAM counters owned by this 'pcifunc' */ 1015 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 1016 1017 /* Delete MCAM entries owned by this 'pcifunc' */ 1018 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) { 1019 if (rule->owner == pcifunc && !rule->default_rule) { 1020 list_del(&rule->list); 1021 kfree(rule); 1022 } 1023 } 1024 1025 mutex_unlock(&mcam->lock); 1026 1027 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1028 } 1029 1030 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 1031 rvu_write64(rvu, blkaddr, \ 1032 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 1033 1034 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 1035 rvu_write64(rvu, blkaddr, \ 1036 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 1037 1038 static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, 1039 struct npc_mcam_kex *mkex, u8 intf) 1040 { 1041 int lid, lt, ld, fl; 1042 1043 if (is_npc_intf_tx(intf)) 1044 return; 1045 1046 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1047 mkex->keyx_cfg[NIX_INTF_RX]); 1048 1049 /* Program LDATA */ 1050 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1051 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1052 for (ld = 0; ld < NPC_MAX_LD; ld++) 1053 SET_KEX_LD(intf, lid, lt, ld, 1054 mkex->intf_lid_lt_ld[NIX_INTF_RX] 1055 [lid][lt][ld]); 1056 } 1057 } 1058 /* Program LFLAGS */ 1059 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1060 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1061 SET_KEX_LDFLAGS(intf, ld, fl, 1062 mkex->intf_ld_flags[NIX_INTF_RX] 1063 [ld][fl]); 1064 } 1065 } 1066 1067 static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, 1068 struct npc_mcam_kex *mkex, u8 intf) 1069 { 1070 int lid, lt, ld, fl; 1071 1072 if (is_npc_intf_rx(intf)) 1073 return; 1074 1075 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1076 mkex->keyx_cfg[NIX_INTF_TX]); 1077 1078 /* Program LDATA */ 1079 for (lid = 0; lid < NPC_MAX_LID; lid++) { 1080 for (lt = 0; lt < NPC_MAX_LT; lt++) { 1081 for (ld = 0; ld < NPC_MAX_LD; ld++) 1082 SET_KEX_LD(intf, lid, lt, ld, 1083 mkex->intf_lid_lt_ld[NIX_INTF_TX] 1084 [lid][lt][ld]); 1085 } 1086 } 1087 /* Program LFLAGS */ 1088 for (ld = 0; ld < NPC_MAX_LD; ld++) { 1089 for (fl = 0; fl < NPC_MAX_LFL; fl++) 1090 SET_KEX_LDFLAGS(intf, ld, fl, 1091 mkex->intf_ld_flags[NIX_INTF_TX] 1092 [ld][fl]); 1093 } 1094 } 1095 1096 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 1097 struct npc_mcam_kex *mkex) 1098 { 1099 struct rvu_hwinfo *hw = rvu->hw; 1100 u8 intf; 1101 int ld; 1102 1103 for (ld = 0; ld < NPC_MAX_LD; ld++) 1104 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 1105 mkex->kex_ld_flags[ld]); 1106 1107 for (intf = 0; intf < hw->npc_intfs; intf++) { 1108 npc_program_mkex_rx(rvu, blkaddr, mkex, intf); 1109 npc_program_mkex_tx(rvu, blkaddr, mkex, intf); 1110 } 1111 } 1112 1113 #define MKEX_END_SIGN 0xdeadbeef 1114 1115 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, 1116 const char *mkex_profile) 1117 { 1118 struct device *dev = &rvu->pdev->dev; 1119 struct npc_mcam_kex *mcam_kex; 1120 void *mkex_prfl_addr = NULL; 1121 u64 prfl_addr, prfl_sz; 1122 1123 /* If user not selected mkex profile */ 1124 if (!strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN)) 1125 goto program_mkex; 1126 1127 if (!rvu->fwdata) 1128 goto program_mkex; 1129 prfl_addr = rvu->fwdata->mcam_addr; 1130 prfl_sz = rvu->fwdata->mcam_sz; 1131 1132 if (!prfl_addr || !prfl_sz) 1133 goto program_mkex; 1134 1135 mkex_prfl_addr = memremap(prfl_addr, prfl_sz, MEMREMAP_WC); 1136 if (!mkex_prfl_addr) 1137 goto program_mkex; 1138 1139 mcam_kex = (struct npc_mcam_kex *)mkex_prfl_addr; 1140 1141 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 1142 /* Compare with mkex mod_param name string */ 1143 if (mcam_kex->mkex_sign == MKEX_SIGN && 1144 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 1145 /* Due to an errata (35786) in A0/B0 pass silicon, 1146 * parse nibble enable configuration has to be 1147 * identical for both Rx and Tx interfaces. 1148 */ 1149 if (!is_rvu_96xx_B0(rvu) || 1150 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) 1151 rvu->kpu.mkex = mcam_kex; 1152 goto program_mkex; 1153 } 1154 1155 mcam_kex++; 1156 prfl_sz -= sizeof(struct npc_mcam_kex); 1157 } 1158 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); 1159 1160 program_mkex: 1161 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); 1162 /* Program selected mkex profile */ 1163 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); 1164 if (mkex_prfl_addr) 1165 memunmap(mkex_prfl_addr); 1166 } 1167 1168 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 1169 const struct npc_kpu_profile_action *kpuaction, 1170 int kpu, int entry, bool pkind) 1171 { 1172 struct npc_kpu_action0 action0 = {0}; 1173 struct npc_kpu_action1 action1 = {0}; 1174 u64 reg; 1175 1176 action1.errlev = kpuaction->errlev; 1177 action1.errcode = kpuaction->errcode; 1178 action1.dp0_offset = kpuaction->dp0_offset; 1179 action1.dp1_offset = kpuaction->dp1_offset; 1180 action1.dp2_offset = kpuaction->dp2_offset; 1181 1182 if (pkind) 1183 reg = NPC_AF_PKINDX_ACTION1(entry); 1184 else 1185 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 1186 1187 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 1188 1189 action0.byp_count = kpuaction->bypass_count; 1190 action0.capture_ena = kpuaction->cap_ena; 1191 action0.parse_done = kpuaction->parse_done; 1192 action0.next_state = kpuaction->next_state; 1193 action0.capture_lid = kpuaction->lid; 1194 action0.capture_ltype = kpuaction->ltype; 1195 action0.capture_flags = kpuaction->flags; 1196 action0.ptr_advance = kpuaction->ptr_advance; 1197 action0.var_len_offset = kpuaction->offset; 1198 action0.var_len_mask = kpuaction->mask; 1199 action0.var_len_right = kpuaction->right; 1200 action0.var_len_shift = kpuaction->shift; 1201 1202 if (pkind) 1203 reg = NPC_AF_PKINDX_ACTION0(entry); 1204 else 1205 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 1206 1207 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 1208 } 1209 1210 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 1211 const struct npc_kpu_profile_cam *kpucam, 1212 int kpu, int entry) 1213 { 1214 struct npc_kpu_cam cam0 = {0}; 1215 struct npc_kpu_cam cam1 = {0}; 1216 1217 cam1.state = kpucam->state & kpucam->state_mask; 1218 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 1219 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 1220 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 1221 1222 cam0.state = ~kpucam->state & kpucam->state_mask; 1223 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 1224 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 1225 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 1226 1227 rvu_write64(rvu, blkaddr, 1228 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 1229 rvu_write64(rvu, blkaddr, 1230 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 1231 } 1232 1233 static inline u64 enable_mask(int count) 1234 { 1235 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 1236 } 1237 1238 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 1239 const struct npc_kpu_profile *profile) 1240 { 1241 int entry, num_entries, max_entries; 1242 1243 if (profile->cam_entries != profile->action_entries) { 1244 dev_err(rvu->dev, 1245 "KPU%d: CAM and action entries [%d != %d] not equal\n", 1246 kpu, profile->cam_entries, profile->action_entries); 1247 } 1248 1249 max_entries = rvu->hw->npc_kpu_entries; 1250 1251 /* Program CAM match entries for previous KPU extracted data */ 1252 num_entries = min_t(int, profile->cam_entries, max_entries); 1253 for (entry = 0; entry < num_entries; entry++) 1254 npc_config_kpucam(rvu, blkaddr, 1255 &profile->cam[entry], kpu, entry); 1256 1257 /* Program this KPU's actions */ 1258 num_entries = min_t(int, profile->action_entries, max_entries); 1259 for (entry = 0; entry < num_entries; entry++) 1260 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 1261 kpu, entry, false); 1262 1263 /* Enable all programmed entries */ 1264 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 1265 rvu_write64(rvu, blkaddr, 1266 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), enable_mask(num_entries)); 1267 if (num_entries > 64) { 1268 rvu_write64(rvu, blkaddr, 1269 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 1270 enable_mask(num_entries - 64)); 1271 } 1272 1273 /* Enable this KPU */ 1274 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 1275 } 1276 1277 static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) 1278 { 1279 profile->name = def_pfl_name; 1280 profile->version = NPC_KPU_PROFILE_VER; 1281 profile->ikpu = ikpu_action_entries; 1282 profile->pkinds = ARRAY_SIZE(ikpu_action_entries); 1283 profile->kpu = npc_kpu_profiles; 1284 profile->kpus = ARRAY_SIZE(npc_kpu_profiles); 1285 profile->lt_def = &npc_lt_defaults; 1286 profile->mkex = &npc_mkex_default; 1287 1288 return 0; 1289 } 1290 1291 static void npc_load_kpu_profile(struct rvu *rvu) 1292 { 1293 struct npc_kpu_profile_adapter *profile = &rvu->kpu; 1294 1295 npc_prepare_default_kpu(profile); 1296 } 1297 1298 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 1299 { 1300 struct rvu_hwinfo *hw = rvu->hw; 1301 int num_pkinds, num_kpus, idx; 1302 struct npc_pkind *pkind; 1303 1304 /* Disable all KPUs and their entries */ 1305 for (idx = 0; idx < hw->npc_kpus; idx++) { 1306 rvu_write64(rvu, blkaddr, 1307 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 1308 rvu_write64(rvu, blkaddr, 1309 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 1310 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 1311 } 1312 1313 /* Load and customize KPU profile. */ 1314 npc_load_kpu_profile(rvu); 1315 1316 /* First program IKPU profile i.e PKIND configs. 1317 * Check HW max count to avoid configuring junk or 1318 * writing to unsupported CSR addresses. 1319 */ 1320 pkind = &hw->pkind; 1321 num_pkinds = rvu->kpu.pkinds; 1322 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 1323 1324 for (idx = 0; idx < num_pkinds; idx++) 1325 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true); 1326 1327 /* Program KPU CAM and Action profiles */ 1328 num_kpus = rvu->kpu.kpus; 1329 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 1330 1331 for (idx = 0; idx < num_kpus; idx++) 1332 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]); 1333 } 1334 1335 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 1336 { 1337 int nixlf_count = rvu_get_nixlf_count(rvu); 1338 struct npc_mcam *mcam = &rvu->hw->mcam; 1339 int rsvd, err; 1340 u64 cfg; 1341 1342 /* Actual number of MCAM entries vary by entry size */ 1343 cfg = (rvu_read64(rvu, blkaddr, 1344 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1345 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1346 mcam->keysize = cfg; 1347 1348 /* Number of banks combined per MCAM entry */ 1349 if (cfg == NPC_MCAM_KEY_X4) 1350 mcam->banks_per_entry = 4; 1351 else if (cfg == NPC_MCAM_KEY_X2) 1352 mcam->banks_per_entry = 2; 1353 else 1354 mcam->banks_per_entry = 1; 1355 1356 /* Reserve one MCAM entry for each of the NIX LF to 1357 * guarantee space to install default matching DMAC rule. 1358 * Also reserve 2 MCAM entries for each PF for default 1359 * channel based matching or 'bcast & promisc' matching to 1360 * support BCAST and PROMISC modes of operation for PFs. 1361 * PF0 is excluded. 1362 */ 1363 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1364 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1365 if (mcam->total_entries <= rsvd) { 1366 dev_warn(rvu->dev, 1367 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1368 mcam->total_entries); 1369 return -ENOMEM; 1370 } 1371 1372 mcam->bmap_entries = mcam->total_entries - rsvd; 1373 mcam->nixlf_offset = mcam->bmap_entries; 1374 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1375 1376 /* Allocate bitmaps for managing MCAM entries */ 1377 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1378 sizeof(long), GFP_KERNEL); 1379 if (!mcam->bmap) 1380 return -ENOMEM; 1381 1382 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1383 BITS_TO_LONGS(mcam->bmap_entries), 1384 sizeof(long), GFP_KERNEL); 1385 if (!mcam->bmap_reverse) 1386 return -ENOMEM; 1387 1388 mcam->bmap_fcnt = mcam->bmap_entries; 1389 1390 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1391 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1392 sizeof(u16), GFP_KERNEL); 1393 if (!mcam->entry2pfvf_map) 1394 return -ENOMEM; 1395 1396 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1397 * allocations and another 1/8th at the top for high priority 1398 * allocations. 1399 */ 1400 mcam->lprio_count = mcam->bmap_entries / 8; 1401 if (mcam->lprio_count > BITS_PER_LONG) 1402 mcam->lprio_count = round_down(mcam->lprio_count, 1403 BITS_PER_LONG); 1404 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1405 mcam->hprio_count = mcam->lprio_count; 1406 mcam->hprio_end = mcam->hprio_count; 1407 1408 1409 /* Allocate bitmap for managing MCAM counters and memory 1410 * for saving counter to RVU PFFUNC allocation mapping. 1411 */ 1412 err = rvu_alloc_bitmap(&mcam->counters); 1413 if (err) 1414 return err; 1415 1416 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1417 sizeof(u16), GFP_KERNEL); 1418 if (!mcam->cntr2pfvf_map) 1419 goto free_mem; 1420 1421 /* Alloc memory for MCAM entry to counter mapping and for tracking 1422 * counter's reference count. 1423 */ 1424 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1425 sizeof(u16), GFP_KERNEL); 1426 if (!mcam->entry2cntr_map) 1427 goto free_mem; 1428 1429 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1430 sizeof(u16), GFP_KERNEL); 1431 if (!mcam->cntr_refcnt) 1432 goto free_mem; 1433 1434 /* Alloc memory for saving target device of mcam rule */ 1435 mcam->entry2target_pffunc = devm_kcalloc(rvu->dev, mcam->total_entries, 1436 sizeof(u16), GFP_KERNEL); 1437 if (!mcam->entry2target_pffunc) 1438 goto free_mem; 1439 1440 mutex_init(&mcam->lock); 1441 1442 return 0; 1443 1444 free_mem: 1445 kfree(mcam->counters.bmap); 1446 return -ENOMEM; 1447 } 1448 1449 static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) 1450 { 1451 struct npc_pkind *pkind = &rvu->hw->pkind; 1452 struct npc_mcam *mcam = &rvu->hw->mcam; 1453 struct rvu_hwinfo *hw = rvu->hw; 1454 u64 npc_const, npc_const1; 1455 u64 npc_const2 = 0; 1456 1457 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1458 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); 1459 if (npc_const1 & BIT_ULL(63)) 1460 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); 1461 1462 pkind->rsrc.max = (npc_const1 >> 12) & 0xFFULL; 1463 hw->npc_kpu_entries = npc_const1 & 0xFFFULL; 1464 hw->npc_kpus = (npc_const >> 8) & 0x1FULL; 1465 hw->npc_intfs = npc_const & 0xFULL; 1466 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; 1467 1468 mcam->banks = (npc_const >> 44) & 0xFULL; 1469 mcam->banksize = (npc_const >> 28) & 0xFFFFULL; 1470 /* Extended set */ 1471 if (npc_const2) { 1472 hw->npc_ext_set = true; 1473 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL; 1474 mcam->banksize = npc_const2 & 0xFFFFULL; 1475 } 1476 1477 mcam->counters.max = hw->npc_counters; 1478 } 1479 1480 static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) 1481 { 1482 struct npc_mcam *mcam = &rvu->hw->mcam; 1483 struct rvu_hwinfo *hw = rvu->hw; 1484 u64 nibble_ena, rx_kex, tx_kex; 1485 u8 intf; 1486 1487 /* Reserve last counter for MCAM RX miss action which is set to 1488 * drop packet. This way we will know how many pkts didn't match 1489 * any MCAM entry. 1490 */ 1491 mcam->counters.max--; 1492 mcam->rx_miss_act_cntr = mcam->counters.max; 1493 1494 rx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_RX]; 1495 tx_kex = npc_mkex_default.keyx_cfg[NIX_INTF_TX]; 1496 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); 1497 1498 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); 1499 if (nibble_ena) { 1500 tx_kex &= ~NPC_PARSE_NIBBLE; 1501 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); 1502 npc_mkex_default.keyx_cfg[NIX_INTF_TX] = tx_kex; 1503 } 1504 1505 /* Configure RX interfaces */ 1506 for (intf = 0; intf < hw->npc_intfs; intf++) { 1507 if (is_npc_intf_tx(intf)) 1508 continue; 1509 1510 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */ 1511 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1512 rx_kex); 1513 1514 /* If MCAM lookup doesn't result in a match, drop the received 1515 * packet. And map this action to a counter to count dropped 1516 * packets. 1517 */ 1518 rvu_write64(rvu, blkaddr, 1519 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); 1520 1521 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] 1522 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] 1523 */ 1524 rvu_write64(rvu, blkaddr, 1525 NPC_AF_INTFX_MISS_STAT_ACT(intf), 1526 ((mcam->rx_miss_act_cntr >> 9) << 12) | 1527 BIT_ULL(9) | mcam->rx_miss_act_cntr); 1528 } 1529 1530 /* Configure TX interfaces */ 1531 for (intf = 0; intf < hw->npc_intfs; intf++) { 1532 if (is_npc_intf_rx(intf)) 1533 continue; 1534 1535 /* Extract Ltypes LID_LA to LID_LE */ 1536 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), 1537 tx_kex); 1538 1539 /* Set TX miss action to UCAST_DEFAULT i.e 1540 * transmit the packet on NIX LF SQ's default channel. 1541 */ 1542 rvu_write64(rvu, blkaddr, 1543 NPC_AF_INTFX_MISS_ACT(intf), 1544 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1545 } 1546 } 1547 1548 int rvu_npc_init(struct rvu *rvu) 1549 { 1550 struct npc_kpu_profile_adapter *kpu = &rvu->kpu; 1551 struct npc_pkind *pkind = &rvu->hw->pkind; 1552 struct npc_mcam *mcam = &rvu->hw->mcam; 1553 int blkaddr, entry, bank, err; 1554 1555 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1556 if (blkaddr < 0) { 1557 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1558 return -ENODEV; 1559 } 1560 1561 rvu_npc_hw_init(rvu, blkaddr); 1562 1563 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1564 for (bank = 0; bank < mcam->banks; bank++) { 1565 for (entry = 0; entry < mcam->banksize; entry++) 1566 rvu_write64(rvu, blkaddr, 1567 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1568 } 1569 1570 err = rvu_alloc_bitmap(&pkind->rsrc); 1571 if (err) 1572 return err; 1573 1574 /* Allocate mem for pkind to PF and channel mapping info */ 1575 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 1576 sizeof(u32), GFP_KERNEL); 1577 if (!pkind->pfchan_map) 1578 return -ENOMEM; 1579 1580 /* Configure KPU profile */ 1581 npc_parser_profile_init(rvu, blkaddr); 1582 1583 /* Config Outer L2, IPv4's NPC layer info */ 1584 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1585 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) | 1586 kpu->lt_def->pck_ol2.ltype_mask); 1587 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1588 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) | 1589 kpu->lt_def->pck_oip4.ltype_mask); 1590 1591 /* Config Inner IPV4 NPC layer info */ 1592 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 1593 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) | 1594 kpu->lt_def->pck_iip4.ltype_mask); 1595 1596 /* Enable below for Rx pkts. 1597 * - Outer IPv4 header checksum validation. 1598 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 1599 * - Inner IPv4 header checksum validation. 1600 * - Set non zero checksum error code value 1601 */ 1602 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 1603 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 1604 BIT_ULL(32) | BIT_ULL(24) | BIT_ULL(6) | 1605 BIT_ULL(2) | BIT_ULL(1)); 1606 1607 rvu_npc_setup_interfaces(rvu, blkaddr); 1608 1609 /* Configure MKEX profile */ 1610 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); 1611 1612 err = npc_mcam_rsrcs_init(rvu, blkaddr); 1613 if (err) 1614 return err; 1615 1616 err = npc_flow_steering_init(rvu, blkaddr); 1617 if (err) { 1618 dev_err(rvu->dev, 1619 "Incorrect mkex profile loaded using default mkex\n"); 1620 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); 1621 } 1622 1623 return 0; 1624 } 1625 1626 void rvu_npc_freemem(struct rvu *rvu) 1627 { 1628 struct npc_pkind *pkind = &rvu->hw->pkind; 1629 struct npc_mcam *mcam = &rvu->hw->mcam; 1630 1631 kfree(pkind->rsrc.bmap); 1632 kfree(mcam->counters.bmap); 1633 mutex_destroy(&mcam->lock); 1634 } 1635 1636 void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, 1637 int blkaddr, int *alloc_cnt, 1638 int *enable_cnt) 1639 { 1640 struct npc_mcam *mcam = &rvu->hw->mcam; 1641 int entry; 1642 1643 *alloc_cnt = 0; 1644 *enable_cnt = 0; 1645 1646 for (entry = 0; entry < mcam->bmap_entries; entry++) { 1647 if (mcam->entry2pfvf_map[entry] == pcifunc) { 1648 (*alloc_cnt)++; 1649 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry)) 1650 (*enable_cnt)++; 1651 } 1652 } 1653 } 1654 1655 void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, 1656 int blkaddr, int *alloc_cnt, 1657 int *enable_cnt) 1658 { 1659 struct npc_mcam *mcam = &rvu->hw->mcam; 1660 int cntr; 1661 1662 *alloc_cnt = 0; 1663 *enable_cnt = 0; 1664 1665 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1666 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1667 (*alloc_cnt)++; 1668 if (mcam->cntr_refcnt[cntr]) 1669 (*enable_cnt)++; 1670 } 1671 } 1672 } 1673 1674 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 1675 u16 pcifunc, int entry) 1676 { 1677 /* Verify if entry is valid and if it is indeed 1678 * allocated to the requesting PFFUNC. 1679 */ 1680 if (entry >= mcam->bmap_entries) 1681 return NPC_MCAM_INVALID_REQ; 1682 1683 if (pcifunc != mcam->entry2pfvf_map[entry]) 1684 return NPC_MCAM_PERM_DENIED; 1685 1686 return 0; 1687 } 1688 1689 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 1690 u16 pcifunc, int cntr) 1691 { 1692 /* Verify if counter is valid and if it is indeed 1693 * allocated to the requesting PFFUNC. 1694 */ 1695 if (cntr >= mcam->counters.max) 1696 return NPC_MCAM_INVALID_REQ; 1697 1698 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 1699 return NPC_MCAM_PERM_DENIED; 1700 1701 return 0; 1702 } 1703 1704 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 1705 int blkaddr, u16 entry, u16 cntr) 1706 { 1707 u16 index = entry & (mcam->banksize - 1); 1708 u16 bank = npc_get_bank(mcam, entry); 1709 1710 /* Set mapping and increment counter's refcnt */ 1711 mcam->entry2cntr_map[entry] = cntr; 1712 mcam->cntr_refcnt[cntr]++; 1713 /* Enable stats 1714 * NPC_AF_MCAMEX_BANKX_STAT_ACT[14:12] - counter[11:9] 1715 * NPC_AF_MCAMEX_BANKX_STAT_ACT[8:0] - counter[8:0] 1716 */ 1717 rvu_write64(rvu, blkaddr, 1718 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 1719 ((cntr >> 9) << 12) | BIT_ULL(9) | cntr); 1720 } 1721 1722 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 1723 struct npc_mcam *mcam, 1724 int blkaddr, u16 entry, u16 cntr) 1725 { 1726 u16 index = entry & (mcam->banksize - 1); 1727 u16 bank = npc_get_bank(mcam, entry); 1728 1729 /* Remove mapping and reduce counter's refcnt */ 1730 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 1731 mcam->cntr_refcnt[cntr]--; 1732 /* Disable stats */ 1733 rvu_write64(rvu, blkaddr, 1734 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 1735 } 1736 1737 /* Sets MCAM entry in bitmap as used. Update 1738 * reverse bitmap too. Should be called with 1739 * 'mcam->lock' held. 1740 */ 1741 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 1742 { 1743 u16 entry, rentry; 1744 1745 entry = index; 1746 rentry = mcam->bmap_entries - index - 1; 1747 1748 __set_bit(entry, mcam->bmap); 1749 __set_bit(rentry, mcam->bmap_reverse); 1750 mcam->bmap_fcnt--; 1751 } 1752 1753 /* Sets MCAM entry in bitmap as free. Update 1754 * reverse bitmap too. Should be called with 1755 * 'mcam->lock' held. 1756 */ 1757 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 1758 { 1759 u16 entry, rentry; 1760 1761 entry = index; 1762 rentry = mcam->bmap_entries - index - 1; 1763 1764 __clear_bit(entry, mcam->bmap); 1765 __clear_bit(rentry, mcam->bmap_reverse); 1766 mcam->bmap_fcnt++; 1767 } 1768 1769 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 1770 int blkaddr, u16 pcifunc) 1771 { 1772 u16 index, cntr; 1773 1774 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 1775 for (index = 0; index < mcam->bmap_entries; index++) { 1776 if (mcam->entry2pfvf_map[index] == pcifunc) { 1777 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1778 /* Free the entry in bitmap */ 1779 npc_mcam_clear_bit(mcam, index); 1780 /* Disable the entry */ 1781 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 1782 1783 /* Update entry2counter mapping */ 1784 cntr = mcam->entry2cntr_map[index]; 1785 if (cntr != NPC_MCAM_INVALID_MAP) 1786 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 1787 blkaddr, index, 1788 cntr); 1789 mcam->entry2target_pffunc[index] = 0x0; 1790 } 1791 } 1792 } 1793 1794 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 1795 u16 pcifunc) 1796 { 1797 u16 cntr; 1798 1799 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 1800 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1801 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1802 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1803 mcam->cntr_refcnt[cntr] = 0; 1804 rvu_free_rsrc(&mcam->counters, cntr); 1805 /* This API is expected to be called after freeing 1806 * MCAM entries, which inturn will remove 1807 * 'entry to counter' mapping. 1808 * No need to do it again. 1809 */ 1810 } 1811 } 1812 } 1813 1814 /* Find area of contiguous free entries of size 'nr'. 1815 * If not found return max contiguous free entries available. 1816 */ 1817 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 1818 u16 nr, u16 *max_area) 1819 { 1820 u16 max_area_start = 0; 1821 u16 index, next, end; 1822 1823 *max_area = 0; 1824 1825 again: 1826 index = find_next_zero_bit(map, size, start); 1827 if (index >= size) 1828 return max_area_start; 1829 1830 end = ((index + nr) >= size) ? size : index + nr; 1831 next = find_next_bit(map, end, index); 1832 if (*max_area < (next - index)) { 1833 *max_area = next - index; 1834 max_area_start = index; 1835 } 1836 1837 if (next < end) { 1838 start = next + 1; 1839 goto again; 1840 } 1841 1842 return max_area_start; 1843 } 1844 1845 /* Find number of free MCAM entries available 1846 * within range i.e in between 'start' and 'end'. 1847 */ 1848 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 1849 { 1850 u16 index, next; 1851 u16 fcnt = 0; 1852 1853 again: 1854 if (start >= end) 1855 return fcnt; 1856 1857 index = find_next_zero_bit(map, end, start); 1858 if (index >= end) 1859 return fcnt; 1860 1861 next = find_next_bit(map, end, index); 1862 if (next <= end) { 1863 fcnt += next - index; 1864 start = next + 1; 1865 goto again; 1866 } 1867 1868 fcnt += end - index; 1869 return fcnt; 1870 } 1871 1872 static void 1873 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 1874 struct npc_mcam_alloc_entry_req *req, 1875 u16 *start, u16 *end, bool *reverse) 1876 { 1877 u16 fcnt; 1878 1879 if (req->priority == NPC_MCAM_HIGHER_PRIO) 1880 goto hprio; 1881 1882 /* For a low priority entry allocation 1883 * - If reference entry is not in hprio zone then 1884 * search range: ref_entry to end. 1885 * - If reference entry is in hprio zone and if 1886 * request can be accomodated in non-hprio zone then 1887 * search range: 'start of middle zone' to 'end' 1888 * - else search in reverse, so that less number of hprio 1889 * zone entries are allocated. 1890 */ 1891 1892 *reverse = false; 1893 *start = req->ref_entry + 1; 1894 *end = mcam->bmap_entries; 1895 1896 if (req->ref_entry >= mcam->hprio_end) 1897 return; 1898 1899 fcnt = npc_mcam_get_free_count(mcam->bmap, 1900 mcam->hprio_end, mcam->bmap_entries); 1901 if (fcnt > req->count) 1902 *start = mcam->hprio_end; 1903 else 1904 *reverse = true; 1905 return; 1906 1907 hprio: 1908 /* For a high priority entry allocation, search is always 1909 * in reverse to preserve hprio zone entries. 1910 * - If reference entry is not in lprio zone then 1911 * search range: 0 to ref_entry. 1912 * - If reference entry is in lprio zone and if 1913 * request can be accomodated in middle zone then 1914 * search range: 'hprio_end' to 'lprio_start' 1915 */ 1916 1917 *reverse = true; 1918 *start = 0; 1919 *end = req->ref_entry; 1920 1921 if (req->ref_entry <= mcam->lprio_start) 1922 return; 1923 1924 fcnt = npc_mcam_get_free_count(mcam->bmap, 1925 mcam->hprio_end, mcam->lprio_start); 1926 if (fcnt < req->count) 1927 return; 1928 *start = mcam->hprio_end; 1929 *end = mcam->lprio_start; 1930 } 1931 1932 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 1933 struct npc_mcam_alloc_entry_req *req, 1934 struct npc_mcam_alloc_entry_rsp *rsp) 1935 { 1936 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 1937 u16 fcnt, hp_fcnt, lp_fcnt; 1938 u16 start, end, index; 1939 int entry, next_start; 1940 bool reverse = false; 1941 unsigned long *bmap; 1942 u16 max_contig; 1943 1944 mutex_lock(&mcam->lock); 1945 1946 /* Check if there are any free entries */ 1947 if (!mcam->bmap_fcnt) { 1948 mutex_unlock(&mcam->lock); 1949 return NPC_MCAM_ALLOC_FAILED; 1950 } 1951 1952 /* MCAM entries are divided into high priority, middle and 1953 * low priority zones. Idea is to not allocate top and lower 1954 * most entries as much as possible, this is to increase 1955 * probability of honouring priority allocation requests. 1956 * 1957 * Two bitmaps are used for mcam entry management, 1958 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 1959 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 1960 * 1961 * Reverse bitmap is used to allocate entries 1962 * - when a higher priority entry is requested 1963 * - when available free entries are less. 1964 * Lower priority ones out of avaialble free entries are always 1965 * chosen when 'high vs low' question arises. 1966 */ 1967 1968 /* Get the search range for priority allocation request */ 1969 if (req->priority) { 1970 npc_get_mcam_search_range_priority(mcam, req, 1971 &start, &end, &reverse); 1972 goto alloc; 1973 } 1974 1975 /* Find out the search range for non-priority allocation request 1976 * 1977 * Get MCAM free entry count in middle zone. 1978 */ 1979 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 1980 mcam->lprio_start, 1981 mcam->bmap_entries); 1982 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 1983 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 1984 1985 /* Check if request can be accomodated in the middle zone */ 1986 if (fcnt > req->count) { 1987 start = mcam->hprio_end; 1988 end = mcam->lprio_start; 1989 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 1990 /* Expand search zone from half of hprio zone to 1991 * half of lprio zone. 1992 */ 1993 start = mcam->hprio_end / 2; 1994 end = mcam->bmap_entries - (mcam->lprio_count / 2); 1995 reverse = true; 1996 } else { 1997 /* Not enough free entries, search all entries in reverse, 1998 * so that low priority ones will get used up. 1999 */ 2000 reverse = true; 2001 start = 0; 2002 end = mcam->bmap_entries; 2003 } 2004 2005 alloc: 2006 if (reverse) { 2007 bmap = mcam->bmap_reverse; 2008 start = mcam->bmap_entries - start; 2009 end = mcam->bmap_entries - end; 2010 index = start; 2011 start = end; 2012 end = index; 2013 } else { 2014 bmap = mcam->bmap; 2015 } 2016 2017 if (req->contig) { 2018 /* Allocate requested number of contiguous entries, if 2019 * unsuccessful find max contiguous entries available. 2020 */ 2021 index = npc_mcam_find_zero_area(bmap, end, start, 2022 req->count, &max_contig); 2023 rsp->count = max_contig; 2024 if (reverse) 2025 rsp->entry = mcam->bmap_entries - index - max_contig; 2026 else 2027 rsp->entry = index; 2028 } else { 2029 /* Allocate requested number of non-contiguous entries, 2030 * if unsuccessful allocate as many as possible. 2031 */ 2032 rsp->count = 0; 2033 next_start = start; 2034 for (entry = 0; entry < req->count; entry++) { 2035 index = find_next_zero_bit(bmap, end, next_start); 2036 if (index >= end) 2037 break; 2038 2039 next_start = start + (index - start) + 1; 2040 2041 /* Save the entry's index */ 2042 if (reverse) 2043 index = mcam->bmap_entries - index - 1; 2044 entry_list[entry] = index; 2045 rsp->count++; 2046 } 2047 } 2048 2049 /* If allocating requested no of entries is unsucessful, 2050 * expand the search range to full bitmap length and retry. 2051 */ 2052 if (!req->priority && (rsp->count < req->count) && 2053 ((end - start) != mcam->bmap_entries)) { 2054 reverse = true; 2055 start = 0; 2056 end = mcam->bmap_entries; 2057 goto alloc; 2058 } 2059 2060 /* For priority entry allocation requests, if allocation is 2061 * failed then expand search to max possible range and retry. 2062 */ 2063 if (req->priority && rsp->count < req->count) { 2064 if (req->priority == NPC_MCAM_LOWER_PRIO && 2065 (start != (req->ref_entry + 1))) { 2066 start = req->ref_entry + 1; 2067 end = mcam->bmap_entries; 2068 reverse = false; 2069 goto alloc; 2070 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 2071 ((end - start) != req->ref_entry)) { 2072 start = 0; 2073 end = req->ref_entry; 2074 reverse = true; 2075 goto alloc; 2076 } 2077 } 2078 2079 /* Copy MCAM entry indices into mbox response entry_list. 2080 * Requester always expects indices in ascending order, so 2081 * so reverse the list if reverse bitmap is used for allocation. 2082 */ 2083 if (!req->contig && rsp->count) { 2084 index = 0; 2085 for (entry = rsp->count - 1; entry >= 0; entry--) { 2086 if (reverse) 2087 rsp->entry_list[index++] = entry_list[entry]; 2088 else 2089 rsp->entry_list[entry] = entry_list[entry]; 2090 } 2091 } 2092 2093 /* Mark the allocated entries as used and set nixlf mapping */ 2094 for (entry = 0; entry < rsp->count; entry++) { 2095 index = req->contig ? 2096 (rsp->entry + entry) : rsp->entry_list[entry]; 2097 npc_mcam_set_bit(mcam, index); 2098 mcam->entry2pfvf_map[index] = pcifunc; 2099 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 2100 } 2101 2102 /* Update available free count in mbox response */ 2103 rsp->free_count = mcam->bmap_fcnt; 2104 2105 mutex_unlock(&mcam->lock); 2106 return 0; 2107 } 2108 2109 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 2110 struct npc_mcam_alloc_entry_req *req, 2111 struct npc_mcam_alloc_entry_rsp *rsp) 2112 { 2113 struct npc_mcam *mcam = &rvu->hw->mcam; 2114 u16 pcifunc = req->hdr.pcifunc; 2115 int blkaddr; 2116 2117 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2118 if (blkaddr < 0) 2119 return NPC_MCAM_INVALID_REQ; 2120 2121 rsp->entry = NPC_MCAM_ENTRY_INVALID; 2122 rsp->free_count = 0; 2123 2124 /* Check if ref_entry is within range */ 2125 if (req->priority && req->ref_entry >= mcam->bmap_entries) 2126 return NPC_MCAM_INVALID_REQ; 2127 2128 /* ref_entry can't be '0' if requested priority is high. 2129 * Can't be last entry if requested priority is low. 2130 */ 2131 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 2132 ((req->ref_entry == (mcam->bmap_entries - 1)) && 2133 req->priority == NPC_MCAM_LOWER_PRIO)) 2134 return NPC_MCAM_INVALID_REQ; 2135 2136 /* Since list of allocated indices needs to be sent to requester, 2137 * max number of non-contiguous entries per mbox msg is limited. 2138 */ 2139 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) 2140 return NPC_MCAM_INVALID_REQ; 2141 2142 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 2143 if (!is_nixlf_attached(rvu, pcifunc)) 2144 return NPC_MCAM_ALLOC_DENIED; 2145 2146 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 2147 } 2148 2149 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 2150 struct npc_mcam_free_entry_req *req, 2151 struct msg_rsp *rsp) 2152 { 2153 struct npc_mcam *mcam = &rvu->hw->mcam; 2154 u16 pcifunc = req->hdr.pcifunc; 2155 int blkaddr, rc = 0; 2156 u16 cntr; 2157 2158 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2159 if (blkaddr < 0) 2160 return NPC_MCAM_INVALID_REQ; 2161 2162 /* Free request from PFFUNC with no NIXLF attached, ignore */ 2163 if (!is_nixlf_attached(rvu, pcifunc)) 2164 return NPC_MCAM_INVALID_REQ; 2165 2166 mutex_lock(&mcam->lock); 2167 2168 if (req->all) 2169 goto free_all; 2170 2171 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2172 if (rc) 2173 goto exit; 2174 2175 mcam->entry2pfvf_map[req->entry] = 0; 2176 mcam->entry2target_pffunc[req->entry] = 0x0; 2177 npc_mcam_clear_bit(mcam, req->entry); 2178 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2179 2180 /* Update entry2counter mapping */ 2181 cntr = mcam->entry2cntr_map[req->entry]; 2182 if (cntr != NPC_MCAM_INVALID_MAP) 2183 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2184 req->entry, cntr); 2185 2186 goto exit; 2187 2188 free_all: 2189 /* Free up all entries allocated to requesting PFFUNC */ 2190 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 2191 exit: 2192 mutex_unlock(&mcam->lock); 2193 return rc; 2194 } 2195 2196 int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu, 2197 struct npc_mcam_read_entry_req *req, 2198 struct npc_mcam_read_entry_rsp *rsp) 2199 { 2200 struct npc_mcam *mcam = &rvu->hw->mcam; 2201 u16 pcifunc = req->hdr.pcifunc; 2202 int blkaddr, rc; 2203 2204 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2205 if (blkaddr < 0) 2206 return NPC_MCAM_INVALID_REQ; 2207 2208 mutex_lock(&mcam->lock); 2209 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2210 if (!rc) { 2211 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry, 2212 &rsp->entry_data, 2213 &rsp->intf, &rsp->enable); 2214 } 2215 2216 mutex_unlock(&mcam->lock); 2217 return rc; 2218 } 2219 2220 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 2221 struct npc_mcam_write_entry_req *req, 2222 struct msg_rsp *rsp) 2223 { 2224 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2225 struct npc_mcam *mcam = &rvu->hw->mcam; 2226 u16 pcifunc = req->hdr.pcifunc; 2227 u16 channel, chan_mask; 2228 int blkaddr, rc; 2229 u8 nix_intf; 2230 2231 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2232 if (blkaddr < 0) 2233 return NPC_MCAM_INVALID_REQ; 2234 2235 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 2236 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 2237 channel &= chan_mask; 2238 2239 mutex_lock(&mcam->lock); 2240 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2241 if (rc) 2242 goto exit; 2243 2244 if (req->set_cntr && 2245 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 2246 rc = NPC_MCAM_INVALID_REQ; 2247 goto exit; 2248 } 2249 2250 if (!is_npc_interface_valid(rvu, req->intf)) { 2251 rc = NPC_MCAM_INVALID_REQ; 2252 goto exit; 2253 } 2254 2255 if (is_npc_intf_tx(req->intf)) 2256 nix_intf = pfvf->nix_tx_intf; 2257 else 2258 nix_intf = pfvf->nix_rx_intf; 2259 2260 if (npc_mcam_verify_channel(rvu, pcifunc, req->intf, channel)) { 2261 rc = NPC_MCAM_INVALID_REQ; 2262 goto exit; 2263 } 2264 2265 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 2266 pcifunc)) { 2267 rc = NPC_MCAM_INVALID_REQ; 2268 goto exit; 2269 } 2270 2271 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf, 2272 &req->entry_data, req->enable_entry); 2273 2274 if (req->set_cntr) 2275 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2276 req->entry, req->cntr); 2277 2278 rc = 0; 2279 exit: 2280 mutex_unlock(&mcam->lock); 2281 return rc; 2282 } 2283 2284 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 2285 struct npc_mcam_ena_dis_entry_req *req, 2286 struct msg_rsp *rsp) 2287 { 2288 struct npc_mcam *mcam = &rvu->hw->mcam; 2289 u16 pcifunc = req->hdr.pcifunc; 2290 int blkaddr, rc; 2291 2292 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2293 if (blkaddr < 0) 2294 return NPC_MCAM_INVALID_REQ; 2295 2296 mutex_lock(&mcam->lock); 2297 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2298 mutex_unlock(&mcam->lock); 2299 if (rc) 2300 return rc; 2301 2302 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 2303 2304 return 0; 2305 } 2306 2307 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 2308 struct npc_mcam_ena_dis_entry_req *req, 2309 struct msg_rsp *rsp) 2310 { 2311 struct npc_mcam *mcam = &rvu->hw->mcam; 2312 u16 pcifunc = req->hdr.pcifunc; 2313 int blkaddr, rc; 2314 2315 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2316 if (blkaddr < 0) 2317 return NPC_MCAM_INVALID_REQ; 2318 2319 mutex_lock(&mcam->lock); 2320 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 2321 mutex_unlock(&mcam->lock); 2322 if (rc) 2323 return rc; 2324 2325 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 2326 2327 return 0; 2328 } 2329 2330 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 2331 struct npc_mcam_shift_entry_req *req, 2332 struct npc_mcam_shift_entry_rsp *rsp) 2333 { 2334 struct npc_mcam *mcam = &rvu->hw->mcam; 2335 u16 pcifunc = req->hdr.pcifunc; 2336 u16 old_entry, new_entry; 2337 u16 index, cntr; 2338 int blkaddr, rc; 2339 2340 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2341 if (blkaddr < 0) 2342 return NPC_MCAM_INVALID_REQ; 2343 2344 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 2345 return NPC_MCAM_INVALID_REQ; 2346 2347 mutex_lock(&mcam->lock); 2348 for (index = 0; index < req->shift_count; index++) { 2349 old_entry = req->curr_entry[index]; 2350 new_entry = req->new_entry[index]; 2351 2352 /* Check if both old and new entries are valid and 2353 * does belong to this PFFUNC or not. 2354 */ 2355 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 2356 if (rc) 2357 break; 2358 2359 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 2360 if (rc) 2361 break; 2362 2363 /* new_entry should not have a counter mapped */ 2364 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 2365 rc = NPC_MCAM_PERM_DENIED; 2366 break; 2367 } 2368 2369 /* Disable the new_entry */ 2370 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 2371 2372 /* Copy rule from old entry to new entry */ 2373 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 2374 2375 /* Copy counter mapping, if any */ 2376 cntr = mcam->entry2cntr_map[old_entry]; 2377 if (cntr != NPC_MCAM_INVALID_MAP) { 2378 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2379 old_entry, cntr); 2380 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2381 new_entry, cntr); 2382 } 2383 2384 /* Enable new_entry and disable old_entry */ 2385 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 2386 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 2387 } 2388 2389 /* If shift has failed then report the failed index */ 2390 if (index != req->shift_count) { 2391 rc = NPC_MCAM_PERM_DENIED; 2392 rsp->failed_entry_idx = index; 2393 } 2394 2395 mutex_unlock(&mcam->lock); 2396 return rc; 2397 } 2398 2399 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 2400 struct npc_mcam_alloc_counter_req *req, 2401 struct npc_mcam_alloc_counter_rsp *rsp) 2402 { 2403 struct npc_mcam *mcam = &rvu->hw->mcam; 2404 u16 pcifunc = req->hdr.pcifunc; 2405 u16 max_contig, cntr; 2406 int blkaddr, index; 2407 2408 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2409 if (blkaddr < 0) 2410 return NPC_MCAM_INVALID_REQ; 2411 2412 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 2413 if (!is_nixlf_attached(rvu, pcifunc)) 2414 return NPC_MCAM_INVALID_REQ; 2415 2416 /* Since list of allocated counter IDs needs to be sent to requester, 2417 * max number of non-contiguous counters per mbox msg is limited. 2418 */ 2419 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 2420 return NPC_MCAM_INVALID_REQ; 2421 2422 mutex_lock(&mcam->lock); 2423 2424 /* Check if unused counters are available or not */ 2425 if (!rvu_rsrc_free_count(&mcam->counters)) { 2426 mutex_unlock(&mcam->lock); 2427 return NPC_MCAM_ALLOC_FAILED; 2428 } 2429 2430 rsp->count = 0; 2431 2432 if (req->contig) { 2433 /* Allocate requested number of contiguous counters, if 2434 * unsuccessful find max contiguous entries available. 2435 */ 2436 index = npc_mcam_find_zero_area(mcam->counters.bmap, 2437 mcam->counters.max, 0, 2438 req->count, &max_contig); 2439 rsp->count = max_contig; 2440 rsp->cntr = index; 2441 for (cntr = index; cntr < (index + max_contig); cntr++) { 2442 __set_bit(cntr, mcam->counters.bmap); 2443 mcam->cntr2pfvf_map[cntr] = pcifunc; 2444 } 2445 } else { 2446 /* Allocate requested number of non-contiguous counters, 2447 * if unsuccessful allocate as many as possible. 2448 */ 2449 for (cntr = 0; cntr < req->count; cntr++) { 2450 index = rvu_alloc_rsrc(&mcam->counters); 2451 if (index < 0) 2452 break; 2453 rsp->cntr_list[cntr] = index; 2454 rsp->count++; 2455 mcam->cntr2pfvf_map[index] = pcifunc; 2456 } 2457 } 2458 2459 mutex_unlock(&mcam->lock); 2460 return 0; 2461 } 2462 2463 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 2464 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2465 { 2466 struct npc_mcam *mcam = &rvu->hw->mcam; 2467 u16 index, entry = 0; 2468 int blkaddr, err; 2469 2470 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2471 if (blkaddr < 0) 2472 return NPC_MCAM_INVALID_REQ; 2473 2474 mutex_lock(&mcam->lock); 2475 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2476 if (err) { 2477 mutex_unlock(&mcam->lock); 2478 return err; 2479 } 2480 2481 /* Mark counter as free/unused */ 2482 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 2483 rvu_free_rsrc(&mcam->counters, req->cntr); 2484 2485 /* Disable all MCAM entry's stats which are using this counter */ 2486 while (entry < mcam->bmap_entries) { 2487 if (!mcam->cntr_refcnt[req->cntr]) 2488 break; 2489 2490 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2491 if (index >= mcam->bmap_entries) 2492 break; 2493 if (mcam->entry2cntr_map[index] != req->cntr) 2494 continue; 2495 2496 entry = index + 1; 2497 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2498 index, req->cntr); 2499 } 2500 2501 mutex_unlock(&mcam->lock); 2502 return 0; 2503 } 2504 2505 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 2506 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 2507 { 2508 struct npc_mcam *mcam = &rvu->hw->mcam; 2509 u16 index, entry = 0; 2510 int blkaddr, rc; 2511 2512 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2513 if (blkaddr < 0) 2514 return NPC_MCAM_INVALID_REQ; 2515 2516 mutex_lock(&mcam->lock); 2517 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2518 if (rc) 2519 goto exit; 2520 2521 /* Unmap the MCAM entry and counter */ 2522 if (!req->all) { 2523 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2524 if (rc) 2525 goto exit; 2526 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2527 req->entry, req->cntr); 2528 goto exit; 2529 } 2530 2531 /* Disable all MCAM entry's stats which are using this counter */ 2532 while (entry < mcam->bmap_entries) { 2533 if (!mcam->cntr_refcnt[req->cntr]) 2534 break; 2535 2536 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2537 if (index >= mcam->bmap_entries) 2538 break; 2539 if (mcam->entry2cntr_map[index] != req->cntr) 2540 continue; 2541 2542 entry = index + 1; 2543 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2544 index, req->cntr); 2545 } 2546 exit: 2547 mutex_unlock(&mcam->lock); 2548 return rc; 2549 } 2550 2551 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2552 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2553 { 2554 struct npc_mcam *mcam = &rvu->hw->mcam; 2555 int blkaddr, err; 2556 2557 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2558 if (blkaddr < 0) 2559 return NPC_MCAM_INVALID_REQ; 2560 2561 mutex_lock(&mcam->lock); 2562 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2563 mutex_unlock(&mcam->lock); 2564 if (err) 2565 return err; 2566 2567 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 2568 2569 return 0; 2570 } 2571 2572 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 2573 struct npc_mcam_oper_counter_req *req, 2574 struct npc_mcam_oper_counter_rsp *rsp) 2575 { 2576 struct npc_mcam *mcam = &rvu->hw->mcam; 2577 int blkaddr, err; 2578 2579 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2580 if (blkaddr < 0) 2581 return NPC_MCAM_INVALID_REQ; 2582 2583 mutex_lock(&mcam->lock); 2584 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2585 mutex_unlock(&mcam->lock); 2586 if (err) 2587 return err; 2588 2589 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 2590 rsp->stat &= BIT_ULL(48) - 1; 2591 2592 return 0; 2593 } 2594 2595 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 2596 struct npc_mcam_alloc_and_write_entry_req *req, 2597 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 2598 { 2599 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 2600 struct npc_mcam_alloc_counter_req cntr_req; 2601 struct npc_mcam_alloc_counter_rsp cntr_rsp; 2602 struct npc_mcam_alloc_entry_req entry_req; 2603 struct npc_mcam_alloc_entry_rsp entry_rsp; 2604 struct npc_mcam *mcam = &rvu->hw->mcam; 2605 u16 entry = NPC_MCAM_ENTRY_INVALID; 2606 u16 cntr = NPC_MCAM_ENTRY_INVALID; 2607 u16 channel, chan_mask; 2608 int blkaddr, rc; 2609 u8 nix_intf; 2610 2611 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2612 if (blkaddr < 0) 2613 return NPC_MCAM_INVALID_REQ; 2614 2615 if (!is_npc_interface_valid(rvu, req->intf)) 2616 return NPC_MCAM_INVALID_REQ; 2617 2618 chan_mask = req->entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; 2619 channel = req->entry_data.kw[0] & NPC_KEX_CHAN_MASK; 2620 channel &= chan_mask; 2621 2622 if (npc_mcam_verify_channel(rvu, req->hdr.pcifunc, req->intf, channel)) 2623 return NPC_MCAM_INVALID_REQ; 2624 2625 if (npc_mcam_verify_pf_func(rvu, &req->entry_data, req->intf, 2626 req->hdr.pcifunc)) 2627 return NPC_MCAM_INVALID_REQ; 2628 2629 /* Try to allocate a MCAM entry */ 2630 entry_req.hdr.pcifunc = req->hdr.pcifunc; 2631 entry_req.contig = true; 2632 entry_req.priority = req->priority; 2633 entry_req.ref_entry = req->ref_entry; 2634 entry_req.count = 1; 2635 2636 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 2637 &entry_req, &entry_rsp); 2638 if (rc) 2639 return rc; 2640 2641 if (!entry_rsp.count) 2642 return NPC_MCAM_ALLOC_FAILED; 2643 2644 entry = entry_rsp.entry; 2645 2646 if (!req->alloc_cntr) 2647 goto write_entry; 2648 2649 /* Now allocate counter */ 2650 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 2651 cntr_req.contig = true; 2652 cntr_req.count = 1; 2653 2654 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 2655 if (rc) { 2656 /* Free allocated MCAM entry */ 2657 mutex_lock(&mcam->lock); 2658 mcam->entry2pfvf_map[entry] = 0; 2659 npc_mcam_clear_bit(mcam, entry); 2660 mutex_unlock(&mcam->lock); 2661 return rc; 2662 } 2663 2664 cntr = cntr_rsp.cntr; 2665 2666 write_entry: 2667 mutex_lock(&mcam->lock); 2668 2669 if (is_npc_intf_tx(req->intf)) 2670 nix_intf = pfvf->nix_tx_intf; 2671 else 2672 nix_intf = pfvf->nix_rx_intf; 2673 2674 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf, 2675 &req->entry_data, req->enable_entry); 2676 2677 if (req->alloc_cntr) 2678 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 2679 mutex_unlock(&mcam->lock); 2680 2681 rsp->entry = entry; 2682 rsp->cntr = cntr; 2683 2684 return 0; 2685 } 2686 2687 #define GET_KEX_CFG(intf) \ 2688 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 2689 2690 #define GET_KEX_FLAGS(ld) \ 2691 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 2692 2693 #define GET_KEX_LD(intf, lid, lt, ld) \ 2694 rvu_read64(rvu, BLKADDR_NPC, \ 2695 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 2696 2697 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 2698 rvu_read64(rvu, BLKADDR_NPC, \ 2699 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 2700 2701 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 2702 struct npc_get_kex_cfg_rsp *rsp) 2703 { 2704 int lid, lt, ld, fl; 2705 2706 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 2707 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 2708 for (lid = 0; lid < NPC_MAX_LID; lid++) { 2709 for (lt = 0; lt < NPC_MAX_LT; lt++) { 2710 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2711 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 2712 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 2713 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 2714 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 2715 } 2716 } 2717 } 2718 for (ld = 0; ld < NPC_MAX_LD; ld++) 2719 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 2720 2721 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2722 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 2723 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 2724 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 2725 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 2726 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 2727 } 2728 } 2729 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 2730 return 0; 2731 } 2732 2733 bool rvu_npc_write_default_rule(struct rvu *rvu, int blkaddr, int nixlf, 2734 u16 pcifunc, u8 intf, struct mcam_entry *entry, 2735 int *index) 2736 { 2737 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 2738 struct npc_mcam *mcam = &rvu->hw->mcam; 2739 bool enable; 2740 u8 nix_intf; 2741 2742 if (is_npc_intf_tx(intf)) 2743 nix_intf = pfvf->nix_tx_intf; 2744 else 2745 nix_intf = pfvf->nix_rx_intf; 2746 2747 *index = npc_get_nixlf_mcam_index(mcam, pcifunc, 2748 nixlf, NIXLF_UCAST_ENTRY); 2749 /* dont force enable unicast entry */ 2750 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, *index); 2751 npc_config_mcam_entry(rvu, mcam, blkaddr, *index, nix_intf, 2752 entry, enable); 2753 2754 return enable; 2755 } 2756 2757 int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu, 2758 struct msg_req *req, 2759 struct npc_mcam_read_base_rule_rsp *rsp) 2760 { 2761 struct npc_mcam *mcam = &rvu->hw->mcam; 2762 int index, blkaddr, nixlf, rc = 0; 2763 u16 pcifunc = req->hdr.pcifunc; 2764 struct rvu_pfvf *pfvf; 2765 u8 intf, enable; 2766 2767 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2768 if (blkaddr < 0) 2769 return NPC_MCAM_INVALID_REQ; 2770 2771 /* Return the channel number in case of PF */ 2772 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 2773 pfvf = rvu_get_pfvf(rvu, pcifunc); 2774 rsp->entry.kw[0] = pfvf->rx_chan_base; 2775 rsp->entry.kw_mask[0] = 0xFFFULL; 2776 goto out; 2777 } 2778 2779 /* Find the pkt steering rule installed by PF to this VF */ 2780 mutex_lock(&mcam->lock); 2781 for (index = 0; index < mcam->bmap_entries; index++) { 2782 if (mcam->entry2target_pffunc[index] == pcifunc) 2783 goto read_entry; 2784 } 2785 2786 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 2787 if (rc < 0) { 2788 mutex_unlock(&mcam->lock); 2789 goto out; 2790 } 2791 /* Read the default ucast entry if there is no pkt steering rule */ 2792 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 2793 NIXLF_UCAST_ENTRY); 2794 read_entry: 2795 /* Read the mcam entry */ 2796 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf, 2797 &enable); 2798 mutex_unlock(&mcam->lock); 2799 out: 2800 return rc; 2801 } 2802