1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "cgx.h" 20 #include "npc_profile.h" 21 22 #define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ 23 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 24 25 #define NIXLF_UCAST_ENTRY 0 26 #define NIXLF_BCAST_ENTRY 1 27 #define NIXLF_PROMISC_ENTRY 2 28 29 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 30 31 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 32 int blkaddr, u16 pcifunc); 33 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 34 u16 pcifunc); 35 36 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 37 { 38 int blkaddr; 39 u64 val = 0; 40 41 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 42 if (blkaddr < 0) 43 return; 44 45 /* Config CPI base for the PKIND */ 46 val = pkind | 1ULL << 62; 47 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 48 } 49 50 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 51 { 52 struct npc_pkind *pkind = &rvu->hw->pkind; 53 u32 map; 54 int i; 55 56 for (i = 0; i < pkind->rsrc.max; i++) { 57 map = pkind->pfchan_map[i]; 58 if (((map >> 16) & 0x3F) == pf) 59 return i; 60 } 61 return -1; 62 } 63 64 static int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 65 u16 pcifunc, int nixlf, int type) 66 { 67 int pf = rvu_get_pf(pcifunc); 68 int index; 69 70 /* Check if this is for a PF */ 71 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 72 /* Reserved entries exclude PF0 */ 73 pf--; 74 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 75 /* Broadcast address matching entry should be first so 76 * that the packet can be replicated to all VFs. 77 */ 78 if (type == NIXLF_BCAST_ENTRY) 79 return index; 80 else if (type == NIXLF_PROMISC_ENTRY) 81 return index + 1; 82 } 83 84 return (mcam->nixlf_offset + (nixlf * RSVD_MCAM_ENTRIES_PER_NIXLF)); 85 } 86 87 static int npc_get_bank(struct npc_mcam *mcam, int index) 88 { 89 int bank = index / mcam->banksize; 90 91 /* 0,1 & 2,3 banks are combined for this keysize */ 92 if (mcam->keysize == NPC_MCAM_KEY_X2) 93 return bank ? 2 : 0; 94 95 return bank; 96 } 97 98 static bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 99 int blkaddr, int index) 100 { 101 int bank = npc_get_bank(mcam, index); 102 u64 cfg; 103 104 index &= (mcam->banksize - 1); 105 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 106 return (cfg & 1); 107 } 108 109 static void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 110 int blkaddr, int index, bool enable) 111 { 112 int bank = npc_get_bank(mcam, index); 113 int actbank = bank; 114 115 index &= (mcam->banksize - 1); 116 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 117 rvu_write64(rvu, blkaddr, 118 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 119 enable ? 1 : 0); 120 } 121 } 122 123 static void npc_get_keyword(struct mcam_entry *entry, int idx, 124 u64 *cam0, u64 *cam1) 125 { 126 u64 kw_mask = 0x00; 127 128 #define CAM_MASK(n) (BIT_ULL(n) - 1) 129 130 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 131 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 132 * 133 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 134 */ 135 switch (idx) { 136 case 0: 137 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 138 *cam1 = entry->kw[0]; 139 kw_mask = entry->kw_mask[0]; 140 break; 141 case 1: 142 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 143 *cam1 = entry->kw[1] & CAM_MASK(48); 144 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 145 break; 146 case 2: 147 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 148 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 149 */ 150 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 151 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 152 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 153 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 154 break; 155 case 3: 156 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 157 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 158 */ 159 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 160 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 161 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 162 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 163 break; 164 case 4: 165 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 166 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 167 */ 168 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 169 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 170 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 171 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 172 break; 173 case 5: 174 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 175 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 176 */ 177 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 178 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 179 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 180 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 181 break; 182 case 6: 183 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 184 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 185 */ 186 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 187 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 188 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 189 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 190 break; 191 case 7: 192 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 193 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 194 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 195 break; 196 } 197 198 *cam1 &= kw_mask; 199 *cam0 = ~*cam1 & kw_mask; 200 } 201 202 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 203 int blkaddr, int index, u8 intf, 204 struct mcam_entry *entry, bool enable) 205 { 206 int bank = npc_get_bank(mcam, index); 207 int kw = 0, actbank, actindex; 208 u64 cam0, cam1; 209 210 actbank = bank; /* Save bank id, to set action later on */ 211 actindex = index; 212 index &= (mcam->banksize - 1); 213 214 /* CAM1 takes the comparison value and 215 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 216 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 217 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 218 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 219 */ 220 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 221 /* Interface should be set in all banks */ 222 rvu_write64(rvu, blkaddr, 223 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 224 intf); 225 rvu_write64(rvu, blkaddr, 226 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 227 ~intf & 0x3); 228 229 /* Set the match key */ 230 npc_get_keyword(entry, kw, &cam0, &cam1); 231 rvu_write64(rvu, blkaddr, 232 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 233 rvu_write64(rvu, blkaddr, 234 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 235 236 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 237 rvu_write64(rvu, blkaddr, 238 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 239 rvu_write64(rvu, blkaddr, 240 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 241 } 242 243 /* Set 'action' */ 244 rvu_write64(rvu, blkaddr, 245 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 246 247 /* Set TAG 'action' */ 248 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 249 entry->vtag_action); 250 251 /* Enable the entry */ 252 if (enable) 253 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 254 else 255 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 256 } 257 258 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 259 int blkaddr, u16 src, u16 dest) 260 { 261 int dbank = npc_get_bank(mcam, dest); 262 int sbank = npc_get_bank(mcam, src); 263 u64 cfg, sreg, dreg; 264 int bank, i; 265 266 src &= (mcam->banksize - 1); 267 dest &= (mcam->banksize - 1); 268 269 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 270 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 271 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 272 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 273 for (i = 0; i < 6; i++) { 274 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 275 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 276 } 277 } 278 279 /* Copy action */ 280 cfg = rvu_read64(rvu, blkaddr, 281 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 282 rvu_write64(rvu, blkaddr, 283 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 284 285 /* Copy TAG action */ 286 cfg = rvu_read64(rvu, blkaddr, 287 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 288 rvu_write64(rvu, blkaddr, 289 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 290 291 /* Enable or disable */ 292 cfg = rvu_read64(rvu, blkaddr, 293 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 294 rvu_write64(rvu, blkaddr, 295 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 296 } 297 298 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 299 int blkaddr, int index) 300 { 301 int bank = npc_get_bank(mcam, index); 302 303 index &= (mcam->banksize - 1); 304 return rvu_read64(rvu, blkaddr, 305 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 306 } 307 308 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 309 int nixlf, u64 chan, u8 *mac_addr) 310 { 311 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 312 struct npc_mcam *mcam = &rvu->hw->mcam; 313 struct mcam_entry entry = { {0} }; 314 struct nix_rx_action action; 315 int blkaddr, index, kwi; 316 u64 mac = 0; 317 318 /* AF's VFs work in promiscuous mode */ 319 if (is_afvf(pcifunc)) 320 return; 321 322 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 323 if (blkaddr < 0) 324 return; 325 326 for (index = ETH_ALEN - 1; index >= 0; index--) 327 mac |= ((u64)*mac_addr++) << (8 * index); 328 329 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 330 nixlf, NIXLF_UCAST_ENTRY); 331 332 /* Match ingress channel and DMAC */ 333 entry.kw[0] = chan; 334 entry.kw_mask[0] = 0xFFFULL; 335 336 kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); 337 entry.kw[kwi] = mac; 338 entry.kw_mask[kwi] = BIT_ULL(48) - 1; 339 340 /* Don't change the action if entry is already enabled 341 * Otherwise RSS action may get overwritten. 342 */ 343 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 344 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 345 blkaddr, index); 346 } else { 347 *(u64 *)&action = 0x00; 348 action.op = NIX_RX_ACTIONOP_UCAST; 349 action.pf_func = pcifunc; 350 } 351 352 entry.action = *(u64 *)&action; 353 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 354 NIX_INTF_RX, &entry, true); 355 356 /* add VLAN matching, setup action and save entry back for later */ 357 entry.kw[0] |= (NPC_LT_LB_STAG | NPC_LT_LB_CTAG) << 20; 358 entry.kw_mask[0] |= (NPC_LT_LB_STAG & NPC_LT_LB_CTAG) << 20; 359 360 entry.vtag_action = VTAG0_VALID_BIT | 361 FIELD_PREP(VTAG0_TYPE_MASK, 0) | 362 FIELD_PREP(VTAG0_LID_MASK, NPC_LID_LA) | 363 FIELD_PREP(VTAG0_RELPTR_MASK, 12); 364 365 memcpy(&pfvf->entry, &entry, sizeof(entry)); 366 } 367 368 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 369 int nixlf, u64 chan, bool allmulti) 370 { 371 struct npc_mcam *mcam = &rvu->hw->mcam; 372 int blkaddr, ucast_idx, index, kwi; 373 struct mcam_entry entry = { {0} }; 374 struct nix_rx_action action = { }; 375 376 /* Only PF or AF VF can add a promiscuous entry */ 377 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) 378 return; 379 380 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 381 if (blkaddr < 0) 382 return; 383 384 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 385 nixlf, NIXLF_PROMISC_ENTRY); 386 387 entry.kw[0] = chan; 388 entry.kw_mask[0] = 0xFFFULL; 389 390 if (allmulti) { 391 kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); 392 entry.kw[kwi] = BIT_ULL(40); /* LSB bit of 1st byte in DMAC */ 393 entry.kw_mask[kwi] = BIT_ULL(40); 394 } 395 396 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc, 397 nixlf, NIXLF_UCAST_ENTRY); 398 399 /* If the corresponding PF's ucast action is RSS, 400 * use the same action for promisc also 401 */ 402 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx)) 403 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 404 blkaddr, ucast_idx); 405 406 if (action.op != NIX_RX_ACTIONOP_RSS) { 407 *(u64 *)&action = 0x00; 408 action.op = NIX_RX_ACTIONOP_UCAST; 409 action.pf_func = pcifunc; 410 } 411 412 entry.action = *(u64 *)&action; 413 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 414 NIX_INTF_RX, &entry, true); 415 } 416 417 static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, 418 int nixlf, bool enable) 419 { 420 struct npc_mcam *mcam = &rvu->hw->mcam; 421 int blkaddr, index; 422 423 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 424 if (blkaddr < 0) 425 return; 426 427 /* Only PF's have a promiscuous entry */ 428 if (pcifunc & RVU_PFVF_FUNC_MASK) 429 return; 430 431 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 432 nixlf, NIXLF_PROMISC_ENTRY); 433 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 434 } 435 436 void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 437 { 438 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); 439 } 440 441 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 442 { 443 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); 444 } 445 446 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 447 int nixlf, u64 chan) 448 { 449 struct npc_mcam *mcam = &rvu->hw->mcam; 450 struct mcam_entry entry = { {0} }; 451 struct nix_rx_action action; 452 #ifdef MCAST_MCE 453 struct rvu_pfvf *pfvf; 454 #endif 455 int blkaddr, index; 456 457 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 458 if (blkaddr < 0) 459 return; 460 461 /* Only PF can add a bcast match entry */ 462 if (pcifunc & RVU_PFVF_FUNC_MASK) 463 return; 464 #ifdef MCAST_MCE 465 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 466 #endif 467 468 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 469 nixlf, NIXLF_BCAST_ENTRY); 470 471 /* Check for L2B bit and LMAC channel 472 * NOTE: Since MKEX default profile(a reduced version intended to 473 * accommodate more capability but igoring few bits) a stap-gap 474 * approach. 475 * Since we care for L2B which by HRM NPC_PARSE_KEX_S at BIT_POS[25], So 476 * moved to BIT_POS[13], ignoring ERRCODE, ERRLEV as we'll loose out 477 * on capability features needed for CoS (/from ODP PoV) e.g: VLAN, 478 * DSCP. 479 * 480 * Reduced layout of MKEX default profile - 481 * Includes following are (i.e.CHAN, L2/3{B/M}, LA, LB, LC, LD): 482 * 483 * BIT_POS[31:28] : LD 484 * BIT_POS[27:24] : LC 485 * BIT_POS[23:20] : LB 486 * BIT_POS[19:16] : LA 487 * BIT_POS[15:12] : L3B, L3M, L2B, L2M 488 * BIT_POS[11:00] : CHAN 489 * 490 */ 491 entry.kw[0] = BIT_ULL(13) | chan; 492 entry.kw_mask[0] = BIT_ULL(13) | 0xFFFULL; 493 494 *(u64 *)&action = 0x00; 495 #ifdef MCAST_MCE 496 /* Early silicon doesn't support pkt replication, 497 * so install entry with UCAST action, so that PF 498 * receives all broadcast packets. 499 */ 500 action.op = NIX_RX_ACTIONOP_MCAST; 501 action.pf_func = pcifunc; 502 action.index = pfvf->bcast_mce_idx; 503 #else 504 action.op = NIX_RX_ACTIONOP_UCAST; 505 action.pf_func = pcifunc; 506 #endif 507 508 entry.action = *(u64 *)&action; 509 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 510 NIX_INTF_RX, &entry, true); 511 } 512 513 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 514 int group, int alg_idx, int mcam_index) 515 { 516 struct npc_mcam *mcam = &rvu->hw->mcam; 517 struct nix_rx_action action; 518 int blkaddr, index, bank; 519 520 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 521 if (blkaddr < 0) 522 return; 523 524 /* Check if this is for reserved default entry */ 525 if (mcam_index < 0) { 526 if (group != DEFAULT_RSS_CONTEXT_GROUP) 527 return; 528 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 529 nixlf, NIXLF_UCAST_ENTRY); 530 } else { 531 /* TODO: validate this mcam index */ 532 index = mcam_index; 533 } 534 535 if (index >= mcam->total_entries) 536 return; 537 538 bank = npc_get_bank(mcam, index); 539 index &= (mcam->banksize - 1); 540 541 *(u64 *)&action = rvu_read64(rvu, blkaddr, 542 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 543 /* Ignore if no action was set earlier */ 544 if (!*(u64 *)&action) 545 return; 546 547 action.op = NIX_RX_ACTIONOP_RSS; 548 action.pf_func = pcifunc; 549 action.index = group; 550 action.flow_key_alg = alg_idx; 551 552 rvu_write64(rvu, blkaddr, 553 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 554 555 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 556 nixlf, NIXLF_PROMISC_ENTRY); 557 558 /* If PF's promiscuous entry is enabled, 559 * Set RSS action for that entry as well 560 */ 561 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 562 bank = npc_get_bank(mcam, index); 563 index &= (mcam->banksize - 1); 564 565 rvu_write64(rvu, blkaddr, 566 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), 567 *(u64 *)&action); 568 } 569 570 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 571 } 572 573 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 574 int nixlf, bool enable) 575 { 576 struct npc_mcam *mcam = &rvu->hw->mcam; 577 struct nix_rx_action action; 578 int index, bank, blkaddr; 579 580 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 581 if (blkaddr < 0) 582 return; 583 584 /* Ucast MCAM match entry of this PF/VF */ 585 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 586 nixlf, NIXLF_UCAST_ENTRY); 587 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 588 589 /* For PF, ena/dis promisc and bcast MCAM match entries */ 590 if (pcifunc & RVU_PFVF_FUNC_MASK) 591 return; 592 593 /* For bcast, enable/disable only if it's action is not 594 * packet replication, incase if action is replication 595 * then this PF's nixlf is removed from bcast replication 596 * list. 597 */ 598 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 599 nixlf, NIXLF_BCAST_ENTRY); 600 bank = npc_get_bank(mcam, index); 601 *(u64 *)&action = rvu_read64(rvu, blkaddr, 602 NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); 603 if (action.op != NIX_RX_ACTIONOP_MCAST) 604 npc_enable_mcam_entry(rvu, mcam, 605 blkaddr, index, enable); 606 if (enable) 607 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); 608 else 609 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); 610 611 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 612 } 613 614 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 615 { 616 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 617 } 618 619 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 620 { 621 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 622 } 623 624 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 625 { 626 struct npc_mcam *mcam = &rvu->hw->mcam; 627 int blkaddr; 628 629 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 630 if (blkaddr < 0) 631 return; 632 633 mutex_lock(&mcam->lock); 634 635 /* Disable and free all MCAM entries mapped to this 'pcifunc' */ 636 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 637 638 /* Free all MCAM counters mapped to this 'pcifunc' */ 639 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 640 641 mutex_unlock(&mcam->lock); 642 643 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 644 } 645 646 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 647 rvu_write64(rvu, blkaddr, \ 648 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 649 650 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 651 rvu_write64(rvu, blkaddr, \ 652 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 653 654 #define KEX_LD_CFG(bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ 655 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 656 ((flags_ena) << 6) | ((key_ofs) & 0x3F)) 657 658 static void npc_config_ldata_extract(struct rvu *rvu, int blkaddr) 659 { 660 struct npc_mcam *mcam = &rvu->hw->mcam; 661 int lid, ltype; 662 int lid_count; 663 u64 cfg; 664 665 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 666 lid_count = (cfg >> 4) & 0xF; 667 668 /* First clear any existing config i.e 669 * disable LDATA and FLAGS extraction. 670 */ 671 for (lid = 0; lid < lid_count; lid++) { 672 for (ltype = 0; ltype < 16; ltype++) { 673 SET_KEX_LD(NIX_INTF_RX, lid, ltype, 0, 0ULL); 674 SET_KEX_LD(NIX_INTF_RX, lid, ltype, 1, 0ULL); 675 SET_KEX_LD(NIX_INTF_TX, lid, ltype, 0, 0ULL); 676 SET_KEX_LD(NIX_INTF_TX, lid, ltype, 1, 0ULL); 677 678 SET_KEX_LDFLAGS(NIX_INTF_RX, 0, ltype, 0ULL); 679 SET_KEX_LDFLAGS(NIX_INTF_RX, 1, ltype, 0ULL); 680 SET_KEX_LDFLAGS(NIX_INTF_TX, 0, ltype, 0ULL); 681 SET_KEX_LDFLAGS(NIX_INTF_TX, 1, ltype, 0ULL); 682 } 683 } 684 685 if (mcam->keysize != NPC_MCAM_KEY_X2) 686 return; 687 688 /* Default MCAM KEX profile */ 689 /* Layer A: Ethernet: */ 690 691 /* DMAC: 6 bytes, KW1[47:0] */ 692 cfg = KEX_LD_CFG(0x05, 0x0, 0x1, 0x0, NPC_PARSE_RESULT_DMAC_OFFSET); 693 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 0, cfg); 694 695 /* Ethertype: 2 bytes, KW0[47:32] */ 696 cfg = KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x4); 697 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 1, cfg); 698 699 /* Layer B: Single VLAN (CTAG) */ 700 /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ 701 cfg = KEX_LD_CFG(0x03, 0x0, 0x1, 0x0, 0x4); 702 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_CTAG, 0, cfg); 703 704 /* Layer B: Stacked VLAN (STAG|QinQ) */ 705 /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ 706 cfg = KEX_LD_CFG(0x03, 0x4, 0x1, 0x0, 0x4); 707 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_STAG, 0, cfg); 708 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_QINQ, 0, cfg); 709 710 /* Layer C: IPv4 */ 711 /* SIP+DIP: 8 bytes, KW2[63:0] */ 712 cfg = KEX_LD_CFG(0x07, 0xc, 0x1, 0x0, 0x10); 713 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 0, cfg); 714 /* TOS: 1 byte, KW1[63:56] */ 715 cfg = KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0xf); 716 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 1, cfg); 717 718 /* Layer D:UDP */ 719 /* SPORT: 2 bytes, KW3[15:0] */ 720 cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); 721 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 0, cfg); 722 /* DPORT: 2 bytes, KW3[31:16] */ 723 cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); 724 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 1, cfg); 725 726 /* Layer D:TCP */ 727 /* SPORT: 2 bytes, KW3[15:0] */ 728 cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); 729 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 0, cfg); 730 /* DPORT: 2 bytes, KW3[31:16] */ 731 cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); 732 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 1, cfg); 733 } 734 735 static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, 736 struct npc_mcam_kex *mkex) 737 { 738 int lid, lt, ld, fl; 739 740 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), 741 mkex->keyx_cfg[NIX_INTF_RX]); 742 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), 743 mkex->keyx_cfg[NIX_INTF_TX]); 744 745 for (ld = 0; ld < NPC_MAX_LD; ld++) 746 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld), 747 mkex->kex_ld_flags[ld]); 748 749 for (lid = 0; lid < NPC_MAX_LID; lid++) { 750 for (lt = 0; lt < NPC_MAX_LT; lt++) { 751 for (ld = 0; ld < NPC_MAX_LD; ld++) { 752 SET_KEX_LD(NIX_INTF_RX, lid, lt, ld, 753 mkex->intf_lid_lt_ld[NIX_INTF_RX] 754 [lid][lt][ld]); 755 756 SET_KEX_LD(NIX_INTF_TX, lid, lt, ld, 757 mkex->intf_lid_lt_ld[NIX_INTF_TX] 758 [lid][lt][ld]); 759 } 760 } 761 } 762 763 for (ld = 0; ld < NPC_MAX_LD; ld++) { 764 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 765 SET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl, 766 mkex->intf_ld_flags[NIX_INTF_RX] 767 [ld][fl]); 768 769 SET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl, 770 mkex->intf_ld_flags[NIX_INTF_TX] 771 [ld][fl]); 772 } 773 } 774 } 775 776 /* strtoull of "mkexprof" with base:36 */ 777 #define MKEX_SIGN 0x19bbfdbd15f 778 #define MKEX_END_SIGN 0xdeadbeef 779 780 static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr) 781 { 782 const char *mkex_profile = rvu->mkex_pfl_name; 783 struct device *dev = &rvu->pdev->dev; 784 void __iomem *mkex_prfl_addr = NULL; 785 struct npc_mcam_kex *mcam_kex; 786 u64 prfl_addr; 787 u64 prfl_sz; 788 789 /* If user not selected mkex profile */ 790 if (!strncmp(mkex_profile, "default", MKEX_NAME_LEN)) 791 goto load_default; 792 793 if (cgx_get_mkex_prfl_info(&prfl_addr, &prfl_sz)) 794 goto load_default; 795 796 if (!prfl_addr || !prfl_sz) 797 goto load_default; 798 799 mkex_prfl_addr = ioremap_wc(prfl_addr, prfl_sz); 800 if (!mkex_prfl_addr) 801 goto load_default; 802 803 mcam_kex = (struct npc_mcam_kex *)mkex_prfl_addr; 804 805 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) { 806 /* Compare with mkex mod_param name string */ 807 if (mcam_kex->mkex_sign == MKEX_SIGN && 808 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) { 809 /* Due to an errata (35786) in A0 pass silicon, 810 * parse nibble enable configuration has to be 811 * identical for both Rx and Tx interfaces. 812 */ 813 if (is_rvu_9xxx_A0(rvu) && 814 mcam_kex->keyx_cfg[NIX_INTF_RX] != 815 mcam_kex->keyx_cfg[NIX_INTF_TX]) 816 goto load_default; 817 818 /* Program selected mkex profile */ 819 npc_program_mkex_profile(rvu, blkaddr, mcam_kex); 820 821 goto unmap; 822 } 823 824 mcam_kex++; 825 prfl_sz -= sizeof(struct npc_mcam_kex); 826 } 827 dev_warn(dev, "Failed to load requested profile: %s\n", 828 rvu->mkex_pfl_name); 829 830 load_default: 831 dev_info(rvu->dev, "Using default mkex profile\n"); 832 /* Config packet data and flags extraction into PARSE result */ 833 npc_config_ldata_extract(rvu, blkaddr); 834 835 unmap: 836 if (mkex_prfl_addr) 837 iounmap(mkex_prfl_addr); 838 } 839 840 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 841 struct npc_kpu_profile_action *kpuaction, 842 int kpu, int entry, bool pkind) 843 { 844 struct npc_kpu_action0 action0 = {0}; 845 struct npc_kpu_action1 action1 = {0}; 846 u64 reg; 847 848 action1.errlev = kpuaction->errlev; 849 action1.errcode = kpuaction->errcode; 850 action1.dp0_offset = kpuaction->dp0_offset; 851 action1.dp1_offset = kpuaction->dp1_offset; 852 action1.dp2_offset = kpuaction->dp2_offset; 853 854 if (pkind) 855 reg = NPC_AF_PKINDX_ACTION1(entry); 856 else 857 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 858 859 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 860 861 action0.byp_count = kpuaction->bypass_count; 862 action0.capture_ena = kpuaction->cap_ena; 863 action0.parse_done = kpuaction->parse_done; 864 action0.next_state = kpuaction->next_state; 865 action0.capture_lid = kpuaction->lid; 866 action0.capture_ltype = kpuaction->ltype; 867 action0.capture_flags = kpuaction->flags; 868 action0.ptr_advance = kpuaction->ptr_advance; 869 action0.var_len_offset = kpuaction->offset; 870 action0.var_len_mask = kpuaction->mask; 871 action0.var_len_right = kpuaction->right; 872 action0.var_len_shift = kpuaction->shift; 873 874 if (pkind) 875 reg = NPC_AF_PKINDX_ACTION0(entry); 876 else 877 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 878 879 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 880 } 881 882 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 883 struct npc_kpu_profile_cam *kpucam, 884 int kpu, int entry) 885 { 886 struct npc_kpu_cam cam0 = {0}; 887 struct npc_kpu_cam cam1 = {0}; 888 889 cam1.state = kpucam->state & kpucam->state_mask; 890 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 891 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 892 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 893 894 cam0.state = ~kpucam->state & kpucam->state_mask; 895 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 896 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 897 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 898 899 rvu_write64(rvu, blkaddr, 900 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 901 rvu_write64(rvu, blkaddr, 902 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 903 } 904 905 static inline u64 enable_mask(int count) 906 { 907 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 908 } 909 910 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 911 struct npc_kpu_profile *profile) 912 { 913 int entry, num_entries, max_entries; 914 915 if (profile->cam_entries != profile->action_entries) { 916 dev_err(rvu->dev, 917 "KPU%d: CAM and action entries [%d != %d] not equal\n", 918 kpu, profile->cam_entries, profile->action_entries); 919 } 920 921 max_entries = rvu_read64(rvu, blkaddr, NPC_AF_CONST1) & 0xFFF; 922 923 /* Program CAM match entries for previous KPU extracted data */ 924 num_entries = min_t(int, profile->cam_entries, max_entries); 925 for (entry = 0; entry < num_entries; entry++) 926 npc_config_kpucam(rvu, blkaddr, 927 &profile->cam[entry], kpu, entry); 928 929 /* Program this KPU's actions */ 930 num_entries = min_t(int, profile->action_entries, max_entries); 931 for (entry = 0; entry < num_entries; entry++) 932 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 933 kpu, entry, false); 934 935 /* Enable all programmed entries */ 936 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 937 rvu_write64(rvu, blkaddr, 938 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), enable_mask(num_entries)); 939 if (num_entries > 64) { 940 rvu_write64(rvu, blkaddr, 941 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 942 enable_mask(num_entries - 64)); 943 } 944 945 /* Enable this KPU */ 946 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 947 } 948 949 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 950 { 951 struct rvu_hwinfo *hw = rvu->hw; 952 int num_pkinds, num_kpus, idx; 953 struct npc_pkind *pkind; 954 955 /* Get HW limits */ 956 hw->npc_kpus = (rvu_read64(rvu, blkaddr, NPC_AF_CONST) >> 8) & 0x1F; 957 958 /* Disable all KPUs and their entries */ 959 for (idx = 0; idx < hw->npc_kpus; idx++) { 960 rvu_write64(rvu, blkaddr, 961 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 962 rvu_write64(rvu, blkaddr, 963 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 964 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 965 } 966 967 /* First program IKPU profile i.e PKIND configs. 968 * Check HW max count to avoid configuring junk or 969 * writing to unsupported CSR addresses. 970 */ 971 pkind = &hw->pkind; 972 num_pkinds = ARRAY_SIZE(ikpu_action_entries); 973 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 974 975 for (idx = 0; idx < num_pkinds; idx++) 976 npc_config_kpuaction(rvu, blkaddr, 977 &ikpu_action_entries[idx], 0, idx, true); 978 979 /* Program KPU CAM and Action profiles */ 980 num_kpus = ARRAY_SIZE(npc_kpu_profiles); 981 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 982 983 for (idx = 0; idx < num_kpus; idx++) 984 npc_program_kpu_profile(rvu, blkaddr, 985 idx, &npc_kpu_profiles[idx]); 986 } 987 988 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 989 { 990 int nixlf_count = rvu_get_nixlf_count(rvu); 991 struct npc_mcam *mcam = &rvu->hw->mcam; 992 int rsvd, err; 993 u64 cfg; 994 995 /* Get HW limits */ 996 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 997 mcam->banks = (cfg >> 44) & 0xF; 998 mcam->banksize = (cfg >> 28) & 0xFFFF; 999 mcam->counters.max = (cfg >> 48) & 0xFFFF; 1000 1001 /* Actual number of MCAM entries vary by entry size */ 1002 cfg = (rvu_read64(rvu, blkaddr, 1003 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 1004 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 1005 mcam->keysize = cfg; 1006 1007 /* Number of banks combined per MCAM entry */ 1008 if (cfg == NPC_MCAM_KEY_X4) 1009 mcam->banks_per_entry = 4; 1010 else if (cfg == NPC_MCAM_KEY_X2) 1011 mcam->banks_per_entry = 2; 1012 else 1013 mcam->banks_per_entry = 1; 1014 1015 /* Reserve one MCAM entry for each of the NIX LF to 1016 * guarantee space to install default matching DMAC rule. 1017 * Also reserve 2 MCAM entries for each PF for default 1018 * channel based matching or 'bcast & promisc' matching to 1019 * support BCAST and PROMISC modes of operation for PFs. 1020 * PF0 is excluded. 1021 */ 1022 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 1023 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 1024 if (mcam->total_entries <= rsvd) { 1025 dev_warn(rvu->dev, 1026 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 1027 mcam->total_entries); 1028 return -ENOMEM; 1029 } 1030 1031 mcam->bmap_entries = mcam->total_entries - rsvd; 1032 mcam->nixlf_offset = mcam->bmap_entries; 1033 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 1034 1035 /* Allocate bitmaps for managing MCAM entries */ 1036 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 1037 sizeof(long), GFP_KERNEL); 1038 if (!mcam->bmap) 1039 return -ENOMEM; 1040 1041 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 1042 BITS_TO_LONGS(mcam->bmap_entries), 1043 sizeof(long), GFP_KERNEL); 1044 if (!mcam->bmap_reverse) 1045 return -ENOMEM; 1046 1047 mcam->bmap_fcnt = mcam->bmap_entries; 1048 1049 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 1050 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1051 sizeof(u16), GFP_KERNEL); 1052 if (!mcam->entry2pfvf_map) 1053 return -ENOMEM; 1054 1055 /* Reserve 1/8th of MCAM entries at the bottom for low priority 1056 * allocations and another 1/8th at the top for high priority 1057 * allocations. 1058 */ 1059 mcam->lprio_count = mcam->bmap_entries / 8; 1060 if (mcam->lprio_count > BITS_PER_LONG) 1061 mcam->lprio_count = round_down(mcam->lprio_count, 1062 BITS_PER_LONG); 1063 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 1064 mcam->hprio_count = mcam->lprio_count; 1065 mcam->hprio_end = mcam->hprio_count; 1066 1067 /* Allocate bitmap for managing MCAM counters and memory 1068 * for saving counter to RVU PFFUNC allocation mapping. 1069 */ 1070 err = rvu_alloc_bitmap(&mcam->counters); 1071 if (err) 1072 return err; 1073 1074 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 1075 sizeof(u16), GFP_KERNEL); 1076 if (!mcam->cntr2pfvf_map) 1077 goto free_mem; 1078 1079 /* Alloc memory for MCAM entry to counter mapping and for tracking 1080 * counter's reference count. 1081 */ 1082 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 1083 sizeof(u16), GFP_KERNEL); 1084 if (!mcam->entry2cntr_map) 1085 goto free_mem; 1086 1087 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 1088 sizeof(u16), GFP_KERNEL); 1089 if (!mcam->cntr_refcnt) 1090 goto free_mem; 1091 1092 mutex_init(&mcam->lock); 1093 1094 return 0; 1095 1096 free_mem: 1097 kfree(mcam->counters.bmap); 1098 return -ENOMEM; 1099 } 1100 1101 int rvu_npc_init(struct rvu *rvu) 1102 { 1103 struct npc_pkind *pkind = &rvu->hw->pkind; 1104 u64 keyz = NPC_MCAM_KEY_X2; 1105 int blkaddr, entry, bank, err; 1106 u64 cfg, nibble_ena; 1107 1108 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1109 if (blkaddr < 0) { 1110 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 1111 return -ENODEV; 1112 } 1113 1114 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 1115 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 1116 for (bank = 0; bank < ((cfg >> 44) & 0xF); bank++) { 1117 for (entry = 0; entry < ((cfg >> 28) & 0xFFFF); entry++) 1118 rvu_write64(rvu, blkaddr, 1119 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 1120 } 1121 1122 /* Allocate resource bimap for pkind*/ 1123 pkind->rsrc.max = (rvu_read64(rvu, blkaddr, 1124 NPC_AF_CONST1) >> 12) & 0xFF; 1125 err = rvu_alloc_bitmap(&pkind->rsrc); 1126 if (err) 1127 return err; 1128 1129 /* Allocate mem for pkind to PF and channel mapping info */ 1130 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 1131 sizeof(u32), GFP_KERNEL); 1132 if (!pkind->pfchan_map) 1133 return -ENOMEM; 1134 1135 /* Configure KPU profile */ 1136 npc_parser_profile_init(rvu, blkaddr); 1137 1138 /* Config Outer L2, IPv4's NPC layer info */ 1139 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1140 (NPC_LID_LA << 8) | (NPC_LT_LA_ETHER << 4) | 0x0F); 1141 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1142 (NPC_LID_LC << 8) | (NPC_LT_LC_IP << 4) | 0x0F); 1143 1144 /* Config Inner IPV4 NPC layer info */ 1145 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4, 1146 (NPC_LID_LF << 8) | (NPC_LT_LF_TU_IP << 4) | 0x0F); 1147 1148 /* Enable below for Rx pkts. 1149 * - Outer IPv4 header checksum validation. 1150 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 1151 * - Inner IPv4 header checksum validation. 1152 * - Set non zero checksum error code value 1153 */ 1154 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 1155 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 1156 BIT_ULL(32) | BIT_ULL(24) | BIT_ULL(6) | 1157 BIT_ULL(2) | BIT_ULL(1)); 1158 1159 /* Set RX and TX side MCAM search key size. 1160 * LA..LD (ltype only) + Channel 1161 */ 1162 nibble_ena = 0x49247; 1163 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), 1164 ((keyz & 0x3) << 32) | nibble_ena); 1165 /* Due to an errata (35786) in A0 pass silicon, parse nibble enable 1166 * configuration has to be identical for both Rx and Tx interfaces. 1167 */ 1168 if (!is_rvu_9xxx_A0(rvu)) 1169 nibble_ena = (1ULL << 19) - 1; 1170 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), 1171 ((keyz & 0x3) << 32) | nibble_ena); 1172 1173 err = npc_mcam_rsrcs_init(rvu, blkaddr); 1174 if (err) 1175 return err; 1176 1177 /* Configure MKEX profile */ 1178 npc_load_mkex_profile(rvu, blkaddr); 1179 1180 /* Set TX miss action to UCAST_DEFAULT i.e 1181 * transmit the packet on NIX LF SQ's default channel. 1182 */ 1183 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_TX), 1184 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1185 1186 /* If MCAM lookup doesn't result in a match, drop the received packet */ 1187 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_RX), 1188 NIX_RX_ACTIONOP_DROP); 1189 1190 return 0; 1191 } 1192 1193 void rvu_npc_freemem(struct rvu *rvu) 1194 { 1195 struct npc_pkind *pkind = &rvu->hw->pkind; 1196 struct npc_mcam *mcam = &rvu->hw->mcam; 1197 1198 kfree(pkind->rsrc.bmap); 1199 kfree(mcam->counters.bmap); 1200 mutex_destroy(&mcam->lock); 1201 } 1202 1203 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 1204 u16 pcifunc, int entry) 1205 { 1206 /* Verify if entry is valid and if it is indeed 1207 * allocated to the requesting PFFUNC. 1208 */ 1209 if (entry >= mcam->bmap_entries) 1210 return NPC_MCAM_INVALID_REQ; 1211 1212 if (pcifunc != mcam->entry2pfvf_map[entry]) 1213 return NPC_MCAM_PERM_DENIED; 1214 1215 return 0; 1216 } 1217 1218 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 1219 u16 pcifunc, int cntr) 1220 { 1221 /* Verify if counter is valid and if it is indeed 1222 * allocated to the requesting PFFUNC. 1223 */ 1224 if (cntr >= mcam->counters.max) 1225 return NPC_MCAM_INVALID_REQ; 1226 1227 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 1228 return NPC_MCAM_PERM_DENIED; 1229 1230 return 0; 1231 } 1232 1233 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 1234 int blkaddr, u16 entry, u16 cntr) 1235 { 1236 u16 index = entry & (mcam->banksize - 1); 1237 u16 bank = npc_get_bank(mcam, entry); 1238 1239 /* Set mapping and increment counter's refcnt */ 1240 mcam->entry2cntr_map[entry] = cntr; 1241 mcam->cntr_refcnt[cntr]++; 1242 /* Enable stats */ 1243 rvu_write64(rvu, blkaddr, 1244 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 1245 BIT_ULL(9) | cntr); 1246 } 1247 1248 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 1249 struct npc_mcam *mcam, 1250 int blkaddr, u16 entry, u16 cntr) 1251 { 1252 u16 index = entry & (mcam->banksize - 1); 1253 u16 bank = npc_get_bank(mcam, entry); 1254 1255 /* Remove mapping and reduce counter's refcnt */ 1256 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 1257 mcam->cntr_refcnt[cntr]--; 1258 /* Disable stats */ 1259 rvu_write64(rvu, blkaddr, 1260 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 1261 } 1262 1263 /* Sets MCAM entry in bitmap as used. Update 1264 * reverse bitmap too. Should be called with 1265 * 'mcam->lock' held. 1266 */ 1267 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 1268 { 1269 u16 entry, rentry; 1270 1271 entry = index; 1272 rentry = mcam->bmap_entries - index - 1; 1273 1274 __set_bit(entry, mcam->bmap); 1275 __set_bit(rentry, mcam->bmap_reverse); 1276 mcam->bmap_fcnt--; 1277 } 1278 1279 /* Sets MCAM entry in bitmap as free. Update 1280 * reverse bitmap too. Should be called with 1281 * 'mcam->lock' held. 1282 */ 1283 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 1284 { 1285 u16 entry, rentry; 1286 1287 entry = index; 1288 rentry = mcam->bmap_entries - index - 1; 1289 1290 __clear_bit(entry, mcam->bmap); 1291 __clear_bit(rentry, mcam->bmap_reverse); 1292 mcam->bmap_fcnt++; 1293 } 1294 1295 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 1296 int blkaddr, u16 pcifunc) 1297 { 1298 u16 index, cntr; 1299 1300 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 1301 for (index = 0; index < mcam->bmap_entries; index++) { 1302 if (mcam->entry2pfvf_map[index] == pcifunc) { 1303 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1304 /* Free the entry in bitmap */ 1305 npc_mcam_clear_bit(mcam, index); 1306 /* Disable the entry */ 1307 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 1308 1309 /* Update entry2counter mapping */ 1310 cntr = mcam->entry2cntr_map[index]; 1311 if (cntr != NPC_MCAM_INVALID_MAP) 1312 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 1313 blkaddr, index, 1314 cntr); 1315 } 1316 } 1317 } 1318 1319 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 1320 u16 pcifunc) 1321 { 1322 u16 cntr; 1323 1324 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 1325 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1326 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1327 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1328 mcam->cntr_refcnt[cntr] = 0; 1329 rvu_free_rsrc(&mcam->counters, cntr); 1330 /* This API is expected to be called after freeing 1331 * MCAM entries, which inturn will remove 1332 * 'entry to counter' mapping. 1333 * No need to do it again. 1334 */ 1335 } 1336 } 1337 } 1338 1339 /* Find area of contiguous free entries of size 'nr'. 1340 * If not found return max contiguous free entries available. 1341 */ 1342 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 1343 u16 nr, u16 *max_area) 1344 { 1345 u16 max_area_start = 0; 1346 u16 index, next, end; 1347 1348 *max_area = 0; 1349 1350 again: 1351 index = find_next_zero_bit(map, size, start); 1352 if (index >= size) 1353 return max_area_start; 1354 1355 end = ((index + nr) >= size) ? size : index + nr; 1356 next = find_next_bit(map, end, index); 1357 if (*max_area < (next - index)) { 1358 *max_area = next - index; 1359 max_area_start = index; 1360 } 1361 1362 if (next < end) { 1363 start = next + 1; 1364 goto again; 1365 } 1366 1367 return max_area_start; 1368 } 1369 1370 /* Find number of free MCAM entries available 1371 * within range i.e in between 'start' and 'end'. 1372 */ 1373 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 1374 { 1375 u16 index, next; 1376 u16 fcnt = 0; 1377 1378 again: 1379 if (start >= end) 1380 return fcnt; 1381 1382 index = find_next_zero_bit(map, end, start); 1383 if (index >= end) 1384 return fcnt; 1385 1386 next = find_next_bit(map, end, index); 1387 if (next <= end) { 1388 fcnt += next - index; 1389 start = next + 1; 1390 goto again; 1391 } 1392 1393 fcnt += end - index; 1394 return fcnt; 1395 } 1396 1397 static void 1398 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 1399 struct npc_mcam_alloc_entry_req *req, 1400 u16 *start, u16 *end, bool *reverse) 1401 { 1402 u16 fcnt; 1403 1404 if (req->priority == NPC_MCAM_HIGHER_PRIO) 1405 goto hprio; 1406 1407 /* For a low priority entry allocation 1408 * - If reference entry is not in hprio zone then 1409 * search range: ref_entry to end. 1410 * - If reference entry is in hprio zone and if 1411 * request can be accomodated in non-hprio zone then 1412 * search range: 'start of middle zone' to 'end' 1413 * - else search in reverse, so that less number of hprio 1414 * zone entries are allocated. 1415 */ 1416 1417 *reverse = false; 1418 *start = req->ref_entry + 1; 1419 *end = mcam->bmap_entries; 1420 1421 if (req->ref_entry >= mcam->hprio_end) 1422 return; 1423 1424 fcnt = npc_mcam_get_free_count(mcam->bmap, 1425 mcam->hprio_end, mcam->bmap_entries); 1426 if (fcnt > req->count) 1427 *start = mcam->hprio_end; 1428 else 1429 *reverse = true; 1430 return; 1431 1432 hprio: 1433 /* For a high priority entry allocation, search is always 1434 * in reverse to preserve hprio zone entries. 1435 * - If reference entry is not in lprio zone then 1436 * search range: 0 to ref_entry. 1437 * - If reference entry is in lprio zone and if 1438 * request can be accomodated in middle zone then 1439 * search range: 'hprio_end' to 'lprio_start' 1440 */ 1441 1442 *reverse = true; 1443 *start = 0; 1444 *end = req->ref_entry; 1445 1446 if (req->ref_entry <= mcam->lprio_start) 1447 return; 1448 1449 fcnt = npc_mcam_get_free_count(mcam->bmap, 1450 mcam->hprio_end, mcam->lprio_start); 1451 if (fcnt < req->count) 1452 return; 1453 *start = mcam->hprio_end; 1454 *end = mcam->lprio_start; 1455 } 1456 1457 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 1458 struct npc_mcam_alloc_entry_req *req, 1459 struct npc_mcam_alloc_entry_rsp *rsp) 1460 { 1461 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 1462 u16 fcnt, hp_fcnt, lp_fcnt; 1463 u16 start, end, index; 1464 int entry, next_start; 1465 bool reverse = false; 1466 unsigned long *bmap; 1467 u16 max_contig; 1468 1469 mutex_lock(&mcam->lock); 1470 1471 /* Check if there are any free entries */ 1472 if (!mcam->bmap_fcnt) { 1473 mutex_unlock(&mcam->lock); 1474 return NPC_MCAM_ALLOC_FAILED; 1475 } 1476 1477 /* MCAM entries are divided into high priority, middle and 1478 * low priority zones. Idea is to not allocate top and lower 1479 * most entries as much as possible, this is to increase 1480 * probability of honouring priority allocation requests. 1481 * 1482 * Two bitmaps are used for mcam entry management, 1483 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 1484 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 1485 * 1486 * Reverse bitmap is used to allocate entries 1487 * - when a higher priority entry is requested 1488 * - when available free entries are less. 1489 * Lower priority ones out of avaialble free entries are always 1490 * chosen when 'high vs low' question arises. 1491 */ 1492 1493 /* Get the search range for priority allocation request */ 1494 if (req->priority) { 1495 npc_get_mcam_search_range_priority(mcam, req, 1496 &start, &end, &reverse); 1497 goto alloc; 1498 } 1499 1500 /* Find out the search range for non-priority allocation request 1501 * 1502 * Get MCAM free entry count in middle zone. 1503 */ 1504 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 1505 mcam->lprio_start, 1506 mcam->bmap_entries); 1507 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 1508 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 1509 1510 /* Check if request can be accomodated in the middle zone */ 1511 if (fcnt > req->count) { 1512 start = mcam->hprio_end; 1513 end = mcam->lprio_start; 1514 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 1515 /* Expand search zone from half of hprio zone to 1516 * half of lprio zone. 1517 */ 1518 start = mcam->hprio_end / 2; 1519 end = mcam->bmap_entries - (mcam->lprio_count / 2); 1520 reverse = true; 1521 } else { 1522 /* Not enough free entries, search all entries in reverse, 1523 * so that low priority ones will get used up. 1524 */ 1525 reverse = true; 1526 start = 0; 1527 end = mcam->bmap_entries; 1528 } 1529 1530 alloc: 1531 if (reverse) { 1532 bmap = mcam->bmap_reverse; 1533 start = mcam->bmap_entries - start; 1534 end = mcam->bmap_entries - end; 1535 index = start; 1536 start = end; 1537 end = index; 1538 } else { 1539 bmap = mcam->bmap; 1540 } 1541 1542 if (req->contig) { 1543 /* Allocate requested number of contiguous entries, if 1544 * unsuccessful find max contiguous entries available. 1545 */ 1546 index = npc_mcam_find_zero_area(bmap, end, start, 1547 req->count, &max_contig); 1548 rsp->count = max_contig; 1549 if (reverse) 1550 rsp->entry = mcam->bmap_entries - index - max_contig; 1551 else 1552 rsp->entry = index; 1553 } else { 1554 /* Allocate requested number of non-contiguous entries, 1555 * if unsuccessful allocate as many as possible. 1556 */ 1557 rsp->count = 0; 1558 next_start = start; 1559 for (entry = 0; entry < req->count; entry++) { 1560 index = find_next_zero_bit(bmap, end, next_start); 1561 if (index >= end) 1562 break; 1563 1564 next_start = start + (index - start) + 1; 1565 1566 /* Save the entry's index */ 1567 if (reverse) 1568 index = mcam->bmap_entries - index - 1; 1569 entry_list[entry] = index; 1570 rsp->count++; 1571 } 1572 } 1573 1574 /* If allocating requested no of entries is unsucessful, 1575 * expand the search range to full bitmap length and retry. 1576 */ 1577 if (!req->priority && (rsp->count < req->count) && 1578 ((end - start) != mcam->bmap_entries)) { 1579 reverse = true; 1580 start = 0; 1581 end = mcam->bmap_entries; 1582 goto alloc; 1583 } 1584 1585 /* For priority entry allocation requests, if allocation is 1586 * failed then expand search to max possible range and retry. 1587 */ 1588 if (req->priority && rsp->count < req->count) { 1589 if (req->priority == NPC_MCAM_LOWER_PRIO && 1590 (start != (req->ref_entry + 1))) { 1591 start = req->ref_entry + 1; 1592 end = mcam->bmap_entries; 1593 reverse = false; 1594 goto alloc; 1595 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 1596 ((end - start) != req->ref_entry)) { 1597 start = 0; 1598 end = req->ref_entry; 1599 reverse = true; 1600 goto alloc; 1601 } 1602 } 1603 1604 /* Copy MCAM entry indices into mbox response entry_list. 1605 * Requester always expects indices in ascending order, so 1606 * so reverse the list if reverse bitmap is used for allocation. 1607 */ 1608 if (!req->contig && rsp->count) { 1609 index = 0; 1610 for (entry = rsp->count - 1; entry >= 0; entry--) { 1611 if (reverse) 1612 rsp->entry_list[index++] = entry_list[entry]; 1613 else 1614 rsp->entry_list[entry] = entry_list[entry]; 1615 } 1616 } 1617 1618 /* Mark the allocated entries as used and set nixlf mapping */ 1619 for (entry = 0; entry < rsp->count; entry++) { 1620 index = req->contig ? 1621 (rsp->entry + entry) : rsp->entry_list[entry]; 1622 npc_mcam_set_bit(mcam, index); 1623 mcam->entry2pfvf_map[index] = pcifunc; 1624 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1625 } 1626 1627 /* Update available free count in mbox response */ 1628 rsp->free_count = mcam->bmap_fcnt; 1629 1630 mutex_unlock(&mcam->lock); 1631 return 0; 1632 } 1633 1634 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 1635 struct npc_mcam_alloc_entry_req *req, 1636 struct npc_mcam_alloc_entry_rsp *rsp) 1637 { 1638 struct npc_mcam *mcam = &rvu->hw->mcam; 1639 u16 pcifunc = req->hdr.pcifunc; 1640 int blkaddr; 1641 1642 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1643 if (blkaddr < 0) 1644 return NPC_MCAM_INVALID_REQ; 1645 1646 rsp->entry = NPC_MCAM_ENTRY_INVALID; 1647 rsp->free_count = 0; 1648 1649 /* Check if ref_entry is within range */ 1650 if (req->priority && req->ref_entry >= mcam->bmap_entries) 1651 return NPC_MCAM_INVALID_REQ; 1652 1653 /* ref_entry can't be '0' if requested priority is high. 1654 * Can't be last entry if requested priority is low. 1655 */ 1656 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 1657 ((req->ref_entry == (mcam->bmap_entries - 1)) && 1658 req->priority == NPC_MCAM_LOWER_PRIO)) 1659 return NPC_MCAM_INVALID_REQ; 1660 1661 /* Since list of allocated indices needs to be sent to requester, 1662 * max number of non-contiguous entries per mbox msg is limited. 1663 */ 1664 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) 1665 return NPC_MCAM_INVALID_REQ; 1666 1667 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 1668 if (!is_nixlf_attached(rvu, pcifunc)) 1669 return NPC_MCAM_ALLOC_DENIED; 1670 1671 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 1672 } 1673 1674 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 1675 struct npc_mcam_free_entry_req *req, 1676 struct msg_rsp *rsp) 1677 { 1678 struct npc_mcam *mcam = &rvu->hw->mcam; 1679 u16 pcifunc = req->hdr.pcifunc; 1680 int blkaddr, rc = 0; 1681 u16 cntr; 1682 1683 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1684 if (blkaddr < 0) 1685 return NPC_MCAM_INVALID_REQ; 1686 1687 /* Free request from PFFUNC with no NIXLF attached, ignore */ 1688 if (!is_nixlf_attached(rvu, pcifunc)) 1689 return NPC_MCAM_INVALID_REQ; 1690 1691 mutex_lock(&mcam->lock); 1692 1693 if (req->all) 1694 goto free_all; 1695 1696 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1697 if (rc) 1698 goto exit; 1699 1700 mcam->entry2pfvf_map[req->entry] = 0; 1701 npc_mcam_clear_bit(mcam, req->entry); 1702 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1703 1704 /* Update entry2counter mapping */ 1705 cntr = mcam->entry2cntr_map[req->entry]; 1706 if (cntr != NPC_MCAM_INVALID_MAP) 1707 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1708 req->entry, cntr); 1709 1710 goto exit; 1711 1712 free_all: 1713 /* Free up all entries allocated to requesting PFFUNC */ 1714 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1715 exit: 1716 mutex_unlock(&mcam->lock); 1717 return rc; 1718 } 1719 1720 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 1721 struct npc_mcam_write_entry_req *req, 1722 struct msg_rsp *rsp) 1723 { 1724 struct npc_mcam *mcam = &rvu->hw->mcam; 1725 u16 pcifunc = req->hdr.pcifunc; 1726 int blkaddr, rc; 1727 1728 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1729 if (blkaddr < 0) 1730 return NPC_MCAM_INVALID_REQ; 1731 1732 mutex_lock(&mcam->lock); 1733 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1734 if (rc) 1735 goto exit; 1736 1737 if (req->set_cntr && 1738 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 1739 rc = NPC_MCAM_INVALID_REQ; 1740 goto exit; 1741 } 1742 1743 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) { 1744 rc = NPC_MCAM_INVALID_REQ; 1745 goto exit; 1746 } 1747 1748 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, req->intf, 1749 &req->entry_data, req->enable_entry); 1750 1751 if (req->set_cntr) 1752 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1753 req->entry, req->cntr); 1754 1755 rc = 0; 1756 exit: 1757 mutex_unlock(&mcam->lock); 1758 return rc; 1759 } 1760 1761 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 1762 struct npc_mcam_ena_dis_entry_req *req, 1763 struct msg_rsp *rsp) 1764 { 1765 struct npc_mcam *mcam = &rvu->hw->mcam; 1766 u16 pcifunc = req->hdr.pcifunc; 1767 int blkaddr, rc; 1768 1769 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1770 if (blkaddr < 0) 1771 return NPC_MCAM_INVALID_REQ; 1772 1773 mutex_lock(&mcam->lock); 1774 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1775 mutex_unlock(&mcam->lock); 1776 if (rc) 1777 return rc; 1778 1779 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 1780 1781 return 0; 1782 } 1783 1784 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 1785 struct npc_mcam_ena_dis_entry_req *req, 1786 struct msg_rsp *rsp) 1787 { 1788 struct npc_mcam *mcam = &rvu->hw->mcam; 1789 u16 pcifunc = req->hdr.pcifunc; 1790 int blkaddr, rc; 1791 1792 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1793 if (blkaddr < 0) 1794 return NPC_MCAM_INVALID_REQ; 1795 1796 mutex_lock(&mcam->lock); 1797 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1798 mutex_unlock(&mcam->lock); 1799 if (rc) 1800 return rc; 1801 1802 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1803 1804 return 0; 1805 } 1806 1807 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 1808 struct npc_mcam_shift_entry_req *req, 1809 struct npc_mcam_shift_entry_rsp *rsp) 1810 { 1811 struct npc_mcam *mcam = &rvu->hw->mcam; 1812 u16 pcifunc = req->hdr.pcifunc; 1813 u16 old_entry, new_entry; 1814 u16 index, cntr; 1815 int blkaddr, rc; 1816 1817 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1818 if (blkaddr < 0) 1819 return NPC_MCAM_INVALID_REQ; 1820 1821 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 1822 return NPC_MCAM_INVALID_REQ; 1823 1824 mutex_lock(&mcam->lock); 1825 for (index = 0; index < req->shift_count; index++) { 1826 old_entry = req->curr_entry[index]; 1827 new_entry = req->new_entry[index]; 1828 1829 /* Check if both old and new entries are valid and 1830 * does belong to this PFFUNC or not. 1831 */ 1832 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 1833 if (rc) 1834 break; 1835 1836 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 1837 if (rc) 1838 break; 1839 1840 /* new_entry should not have a counter mapped */ 1841 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 1842 rc = NPC_MCAM_PERM_DENIED; 1843 break; 1844 } 1845 1846 /* Disable the new_entry */ 1847 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 1848 1849 /* Copy rule from old entry to new entry */ 1850 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 1851 1852 /* Copy counter mapping, if any */ 1853 cntr = mcam->entry2cntr_map[old_entry]; 1854 if (cntr != NPC_MCAM_INVALID_MAP) { 1855 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1856 old_entry, cntr); 1857 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1858 new_entry, cntr); 1859 } 1860 1861 /* Enable new_entry and disable old_entry */ 1862 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 1863 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 1864 } 1865 1866 /* If shift has failed then report the failed index */ 1867 if (index != req->shift_count) { 1868 rc = NPC_MCAM_PERM_DENIED; 1869 rsp->failed_entry_idx = index; 1870 } 1871 1872 mutex_unlock(&mcam->lock); 1873 return rc; 1874 } 1875 1876 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 1877 struct npc_mcam_alloc_counter_req *req, 1878 struct npc_mcam_alloc_counter_rsp *rsp) 1879 { 1880 struct npc_mcam *mcam = &rvu->hw->mcam; 1881 u16 pcifunc = req->hdr.pcifunc; 1882 u16 max_contig, cntr; 1883 int blkaddr, index; 1884 1885 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1886 if (blkaddr < 0) 1887 return NPC_MCAM_INVALID_REQ; 1888 1889 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 1890 if (!is_nixlf_attached(rvu, pcifunc)) 1891 return NPC_MCAM_INVALID_REQ; 1892 1893 /* Since list of allocated counter IDs needs to be sent to requester, 1894 * max number of non-contiguous counters per mbox msg is limited. 1895 */ 1896 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 1897 return NPC_MCAM_INVALID_REQ; 1898 1899 mutex_lock(&mcam->lock); 1900 1901 /* Check if unused counters are available or not */ 1902 if (!rvu_rsrc_free_count(&mcam->counters)) { 1903 mutex_unlock(&mcam->lock); 1904 return NPC_MCAM_ALLOC_FAILED; 1905 } 1906 1907 rsp->count = 0; 1908 1909 if (req->contig) { 1910 /* Allocate requested number of contiguous counters, if 1911 * unsuccessful find max contiguous entries available. 1912 */ 1913 index = npc_mcam_find_zero_area(mcam->counters.bmap, 1914 mcam->counters.max, 0, 1915 req->count, &max_contig); 1916 rsp->count = max_contig; 1917 rsp->cntr = index; 1918 for (cntr = index; cntr < (index + max_contig); cntr++) { 1919 __set_bit(cntr, mcam->counters.bmap); 1920 mcam->cntr2pfvf_map[cntr] = pcifunc; 1921 } 1922 } else { 1923 /* Allocate requested number of non-contiguous counters, 1924 * if unsuccessful allocate as many as possible. 1925 */ 1926 for (cntr = 0; cntr < req->count; cntr++) { 1927 index = rvu_alloc_rsrc(&mcam->counters); 1928 if (index < 0) 1929 break; 1930 rsp->cntr_list[cntr] = index; 1931 rsp->count++; 1932 mcam->cntr2pfvf_map[index] = pcifunc; 1933 } 1934 } 1935 1936 mutex_unlock(&mcam->lock); 1937 return 0; 1938 } 1939 1940 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 1941 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 1942 { 1943 struct npc_mcam *mcam = &rvu->hw->mcam; 1944 u16 index, entry = 0; 1945 int blkaddr, err; 1946 1947 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1948 if (blkaddr < 0) 1949 return NPC_MCAM_INVALID_REQ; 1950 1951 mutex_lock(&mcam->lock); 1952 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1953 if (err) { 1954 mutex_unlock(&mcam->lock); 1955 return err; 1956 } 1957 1958 /* Mark counter as free/unused */ 1959 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 1960 rvu_free_rsrc(&mcam->counters, req->cntr); 1961 1962 /* Disable all MCAM entry's stats which are using this counter */ 1963 while (entry < mcam->bmap_entries) { 1964 if (!mcam->cntr_refcnt[req->cntr]) 1965 break; 1966 1967 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 1968 if (index >= mcam->bmap_entries) 1969 break; 1970 if (mcam->entry2cntr_map[index] != req->cntr) 1971 continue; 1972 1973 entry = index + 1; 1974 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1975 index, req->cntr); 1976 } 1977 1978 mutex_unlock(&mcam->lock); 1979 return 0; 1980 } 1981 1982 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 1983 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 1984 { 1985 struct npc_mcam *mcam = &rvu->hw->mcam; 1986 u16 index, entry = 0; 1987 int blkaddr, rc; 1988 1989 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1990 if (blkaddr < 0) 1991 return NPC_MCAM_INVALID_REQ; 1992 1993 mutex_lock(&mcam->lock); 1994 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1995 if (rc) 1996 goto exit; 1997 1998 /* Unmap the MCAM entry and counter */ 1999 if (!req->all) { 2000 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 2001 if (rc) 2002 goto exit; 2003 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2004 req->entry, req->cntr); 2005 goto exit; 2006 } 2007 2008 /* Disable all MCAM entry's stats which are using this counter */ 2009 while (entry < mcam->bmap_entries) { 2010 if (!mcam->cntr_refcnt[req->cntr]) 2011 break; 2012 2013 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 2014 if (index >= mcam->bmap_entries) 2015 break; 2016 if (mcam->entry2cntr_map[index] != req->cntr) 2017 continue; 2018 2019 entry = index + 1; 2020 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 2021 index, req->cntr); 2022 } 2023 exit: 2024 mutex_unlock(&mcam->lock); 2025 return rc; 2026 } 2027 2028 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 2029 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 2030 { 2031 struct npc_mcam *mcam = &rvu->hw->mcam; 2032 int blkaddr, err; 2033 2034 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2035 if (blkaddr < 0) 2036 return NPC_MCAM_INVALID_REQ; 2037 2038 mutex_lock(&mcam->lock); 2039 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2040 mutex_unlock(&mcam->lock); 2041 if (err) 2042 return err; 2043 2044 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 2045 2046 return 0; 2047 } 2048 2049 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 2050 struct npc_mcam_oper_counter_req *req, 2051 struct npc_mcam_oper_counter_rsp *rsp) 2052 { 2053 struct npc_mcam *mcam = &rvu->hw->mcam; 2054 int blkaddr, err; 2055 2056 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2057 if (blkaddr < 0) 2058 return NPC_MCAM_INVALID_REQ; 2059 2060 mutex_lock(&mcam->lock); 2061 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 2062 mutex_unlock(&mcam->lock); 2063 if (err) 2064 return err; 2065 2066 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 2067 rsp->stat &= BIT_ULL(48) - 1; 2068 2069 return 0; 2070 } 2071 2072 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 2073 struct npc_mcam_alloc_and_write_entry_req *req, 2074 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 2075 { 2076 struct npc_mcam_alloc_counter_req cntr_req; 2077 struct npc_mcam_alloc_counter_rsp cntr_rsp; 2078 struct npc_mcam_alloc_entry_req entry_req; 2079 struct npc_mcam_alloc_entry_rsp entry_rsp; 2080 struct npc_mcam *mcam = &rvu->hw->mcam; 2081 u16 entry = NPC_MCAM_ENTRY_INVALID; 2082 u16 cntr = NPC_MCAM_ENTRY_INVALID; 2083 int blkaddr, rc; 2084 2085 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2086 if (blkaddr < 0) 2087 return NPC_MCAM_INVALID_REQ; 2088 2089 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) 2090 return NPC_MCAM_INVALID_REQ; 2091 2092 /* Try to allocate a MCAM entry */ 2093 entry_req.hdr.pcifunc = req->hdr.pcifunc; 2094 entry_req.contig = true; 2095 entry_req.priority = req->priority; 2096 entry_req.ref_entry = req->ref_entry; 2097 entry_req.count = 1; 2098 2099 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 2100 &entry_req, &entry_rsp); 2101 if (rc) 2102 return rc; 2103 2104 if (!entry_rsp.count) 2105 return NPC_MCAM_ALLOC_FAILED; 2106 2107 entry = entry_rsp.entry; 2108 2109 if (!req->alloc_cntr) 2110 goto write_entry; 2111 2112 /* Now allocate counter */ 2113 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 2114 cntr_req.contig = true; 2115 cntr_req.count = 1; 2116 2117 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 2118 if (rc) { 2119 /* Free allocated MCAM entry */ 2120 mutex_lock(&mcam->lock); 2121 mcam->entry2pfvf_map[entry] = 0; 2122 npc_mcam_clear_bit(mcam, entry); 2123 mutex_unlock(&mcam->lock); 2124 return rc; 2125 } 2126 2127 cntr = cntr_rsp.cntr; 2128 2129 write_entry: 2130 mutex_lock(&mcam->lock); 2131 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, req->intf, 2132 &req->entry_data, req->enable_entry); 2133 2134 if (req->alloc_cntr) 2135 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 2136 mutex_unlock(&mcam->lock); 2137 2138 rsp->entry = entry; 2139 rsp->cntr = cntr; 2140 2141 return 0; 2142 } 2143 2144 #define GET_KEX_CFG(intf) \ 2145 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 2146 2147 #define GET_KEX_FLAGS(ld) \ 2148 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 2149 2150 #define GET_KEX_LD(intf, lid, lt, ld) \ 2151 rvu_read64(rvu, BLKADDR_NPC, \ 2152 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 2153 2154 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 2155 rvu_read64(rvu, BLKADDR_NPC, \ 2156 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 2157 2158 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 2159 struct npc_get_kex_cfg_rsp *rsp) 2160 { 2161 int lid, lt, ld, fl; 2162 2163 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 2164 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 2165 for (lid = 0; lid < NPC_MAX_LID; lid++) { 2166 for (lt = 0; lt < NPC_MAX_LT; lt++) { 2167 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2168 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 2169 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 2170 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 2171 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 2172 } 2173 } 2174 } 2175 for (ld = 0; ld < NPC_MAX_LD; ld++) 2176 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 2177 2178 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2179 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 2180 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 2181 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 2182 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 2183 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 2184 } 2185 } 2186 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); 2187 return 0; 2188 } 2189 2190 int rvu_npc_update_rxvlan(struct rvu *rvu, u16 pcifunc, int nixlf) 2191 { 2192 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 2193 struct npc_mcam *mcam = &rvu->hw->mcam; 2194 int blkaddr, index; 2195 bool enable; 2196 2197 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2198 if (blkaddr < 0) 2199 return NIX_AF_ERR_AF_LF_INVALID; 2200 2201 if (!pfvf->rxvlan) 2202 return 0; 2203 2204 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 2205 NIXLF_UCAST_ENTRY); 2206 pfvf->entry.action = npc_get_mcam_action(rvu, mcam, blkaddr, index); 2207 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, index); 2208 npc_config_mcam_entry(rvu, mcam, blkaddr, pfvf->rxvlan_index, 2209 NIX_INTF_RX, &pfvf->entry, enable); 2210 2211 return 0; 2212 } 2213