1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/module.h> 13 #include <linux/pci.h> 14 15 #include "rvu_struct.h" 16 #include "rvu_reg.h" 17 #include "rvu.h" 18 #include "npc.h" 19 #include "npc_profile.h" 20 21 #define RSVD_MCAM_ENTRIES_PER_PF 2 /* Bcast & Promisc */ 22 #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ 23 24 #define NIXLF_UCAST_ENTRY 0 25 #define NIXLF_BCAST_ENTRY 1 26 #define NIXLF_PROMISC_ENTRY 2 27 28 #define NPC_PARSE_RESULT_DMAC_OFFSET 8 29 30 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 31 int blkaddr, u16 pcifunc); 32 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 33 u16 pcifunc); 34 35 void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf) 36 { 37 int blkaddr; 38 u64 val = 0; 39 40 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 41 if (blkaddr < 0) 42 return; 43 44 /* Config CPI base for the PKIND */ 45 val = pkind | 1ULL << 62; 46 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val); 47 } 48 49 int rvu_npc_get_pkind(struct rvu *rvu, u16 pf) 50 { 51 struct npc_pkind *pkind = &rvu->hw->pkind; 52 u32 map; 53 int i; 54 55 for (i = 0; i < pkind->rsrc.max; i++) { 56 map = pkind->pfchan_map[i]; 57 if (((map >> 16) & 0x3F) == pf) 58 return i; 59 } 60 return -1; 61 } 62 63 static int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, 64 u16 pcifunc, int nixlf, int type) 65 { 66 int pf = rvu_get_pf(pcifunc); 67 int index; 68 69 /* Check if this is for a PF */ 70 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { 71 /* Reserved entries exclude PF0 */ 72 pf--; 73 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF); 74 /* Broadcast address matching entry should be first so 75 * that the packet can be replicated to all VFs. 76 */ 77 if (type == NIXLF_BCAST_ENTRY) 78 return index; 79 else if (type == NIXLF_PROMISC_ENTRY) 80 return index + 1; 81 } 82 83 return (mcam->nixlf_offset + (nixlf * RSVD_MCAM_ENTRIES_PER_NIXLF)); 84 } 85 86 static int npc_get_bank(struct npc_mcam *mcam, int index) 87 { 88 int bank = index / mcam->banksize; 89 90 /* 0,1 & 2,3 banks are combined for this keysize */ 91 if (mcam->keysize == NPC_MCAM_KEY_X2) 92 return bank ? 2 : 0; 93 94 return bank; 95 } 96 97 static bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, 98 int blkaddr, int index) 99 { 100 int bank = npc_get_bank(mcam, index); 101 u64 cfg; 102 103 index &= (mcam->banksize - 1); 104 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); 105 return (cfg & 1); 106 } 107 108 static void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 109 int blkaddr, int index, bool enable) 110 { 111 int bank = npc_get_bank(mcam, index); 112 int actbank = bank; 113 114 index &= (mcam->banksize - 1); 115 for (; bank < (actbank + mcam->banks_per_entry); bank++) { 116 rvu_write64(rvu, blkaddr, 117 NPC_AF_MCAMEX_BANKX_CFG(index, bank), 118 enable ? 1 : 0); 119 } 120 } 121 122 static void npc_get_keyword(struct mcam_entry *entry, int idx, 123 u64 *cam0, u64 *cam1) 124 { 125 u64 kw_mask = 0x00; 126 127 #define CAM_MASK(n) (BIT_ULL(n) - 1) 128 129 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and 130 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1. 131 * 132 * Also, only 48 bits of BANKX_CAMX_W1 are valid. 133 */ 134 switch (idx) { 135 case 0: 136 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */ 137 *cam1 = entry->kw[0]; 138 kw_mask = entry->kw_mask[0]; 139 break; 140 case 1: 141 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */ 142 *cam1 = entry->kw[1] & CAM_MASK(48); 143 kw_mask = entry->kw_mask[1] & CAM_MASK(48); 144 break; 145 case 2: 146 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48> 147 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0> 148 */ 149 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16); 150 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16); 151 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16); 152 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16); 153 break; 154 case 3: 155 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48> 156 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0> 157 */ 158 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16); 159 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16); 160 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16); 161 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16); 162 break; 163 case 4: 164 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32> 165 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0> 166 */ 167 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32); 168 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32); 169 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32); 170 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32); 171 break; 172 case 5: 173 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32> 174 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0> 175 */ 176 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32); 177 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32); 178 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32); 179 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32); 180 break; 181 case 6: 182 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16> 183 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0> 184 */ 185 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48); 186 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48); 187 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48); 188 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48); 189 break; 190 case 7: 191 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */ 192 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48); 193 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48); 194 break; 195 } 196 197 *cam1 &= kw_mask; 198 *cam0 = ~*cam1 & kw_mask; 199 } 200 201 static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 202 int blkaddr, int index, u8 intf, 203 struct mcam_entry *entry, bool enable) 204 { 205 int bank = npc_get_bank(mcam, index); 206 int kw = 0, actbank, actindex; 207 u64 cam0, cam1; 208 209 actbank = bank; /* Save bank id, to set action later on */ 210 actindex = index; 211 index &= (mcam->banksize - 1); 212 213 /* CAM1 takes the comparison value and 214 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. 215 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 216 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 217 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. 218 */ 219 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) { 220 /* Interface should be set in all banks */ 221 rvu_write64(rvu, blkaddr, 222 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 223 intf); 224 rvu_write64(rvu, blkaddr, 225 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 226 ~intf & 0x3); 227 228 /* Set the match key */ 229 npc_get_keyword(entry, kw, &cam0, &cam1); 230 rvu_write64(rvu, blkaddr, 231 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1); 232 rvu_write64(rvu, blkaddr, 233 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0); 234 235 npc_get_keyword(entry, kw + 1, &cam0, &cam1); 236 rvu_write64(rvu, blkaddr, 237 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1); 238 rvu_write64(rvu, blkaddr, 239 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0); 240 } 241 242 /* Set 'action' */ 243 rvu_write64(rvu, blkaddr, 244 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action); 245 246 /* Set TAG 'action' */ 247 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank), 248 entry->vtag_action); 249 250 /* Enable the entry */ 251 if (enable) 252 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true); 253 else 254 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false); 255 } 256 257 static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, 258 int blkaddr, u16 src, u16 dest) 259 { 260 int dbank = npc_get_bank(mcam, dest); 261 int sbank = npc_get_bank(mcam, src); 262 u64 cfg, sreg, dreg; 263 int bank, i; 264 265 src &= (mcam->banksize - 1); 266 dest &= (mcam->banksize - 1); 267 268 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */ 269 for (bank = 0; bank < mcam->banks_per_entry; bank++) { 270 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0); 271 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0); 272 for (i = 0; i < 6; i++) { 273 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); 274 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); 275 } 276 } 277 278 /* Copy action */ 279 cfg = rvu_read64(rvu, blkaddr, 280 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank)); 281 rvu_write64(rvu, blkaddr, 282 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg); 283 284 /* Copy TAG action */ 285 cfg = rvu_read64(rvu, blkaddr, 286 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank)); 287 rvu_write64(rvu, blkaddr, 288 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg); 289 290 /* Enable or disable */ 291 cfg = rvu_read64(rvu, blkaddr, 292 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)); 293 rvu_write64(rvu, blkaddr, 294 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg); 295 } 296 297 static u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, 298 int blkaddr, int index) 299 { 300 int bank = npc_get_bank(mcam, index); 301 302 index &= (mcam->banksize - 1); 303 return rvu_read64(rvu, blkaddr, 304 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 305 } 306 307 void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, 308 int nixlf, u64 chan, u8 *mac_addr) 309 { 310 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 311 struct npc_mcam *mcam = &rvu->hw->mcam; 312 struct mcam_entry entry = { {0} }; 313 struct nix_rx_action action; 314 int blkaddr, index, kwi; 315 u64 mac = 0; 316 317 /* AF's VFs work in promiscuous mode */ 318 if (is_afvf(pcifunc)) 319 return; 320 321 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 322 if (blkaddr < 0) 323 return; 324 325 for (index = ETH_ALEN - 1; index >= 0; index--) 326 mac |= ((u64)*mac_addr++) << (8 * index); 327 328 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 329 nixlf, NIXLF_UCAST_ENTRY); 330 331 /* Match ingress channel and DMAC */ 332 entry.kw[0] = chan; 333 entry.kw_mask[0] = 0xFFFULL; 334 335 kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); 336 entry.kw[kwi] = mac; 337 entry.kw_mask[kwi] = BIT_ULL(48) - 1; 338 339 /* Don't change the action if entry is already enabled 340 * Otherwise RSS action may get overwritten. 341 */ 342 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) { 343 *(u64 *)&action = npc_get_mcam_action(rvu, mcam, 344 blkaddr, index); 345 } else { 346 *(u64 *)&action = 0x00; 347 action.op = NIX_RX_ACTIONOP_UCAST; 348 action.pf_func = pcifunc; 349 } 350 351 entry.action = *(u64 *)&action; 352 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 353 NIX_INTF_RX, &entry, true); 354 355 /* add VLAN matching, setup action and save entry back for later */ 356 entry.kw[0] |= (NPC_LT_LB_STAG | NPC_LT_LB_CTAG) << 20; 357 entry.kw_mask[0] |= (NPC_LT_LB_STAG & NPC_LT_LB_CTAG) << 20; 358 359 entry.vtag_action = VTAG0_VALID_BIT | 360 FIELD_PREP(VTAG0_TYPE_MASK, 0) | 361 FIELD_PREP(VTAG0_LID_MASK, NPC_LID_LA) | 362 FIELD_PREP(VTAG0_RELPTR_MASK, 12); 363 364 memcpy(&pfvf->entry, &entry, sizeof(entry)); 365 } 366 367 void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, 368 int nixlf, u64 chan, bool allmulti) 369 { 370 struct npc_mcam *mcam = &rvu->hw->mcam; 371 struct mcam_entry entry = { {0} }; 372 struct nix_rx_action action = { }; 373 int blkaddr, index, kwi; 374 375 /* Only PF or AF VF can add a promiscuous entry */ 376 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !is_afvf(pcifunc)) 377 return; 378 379 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 380 if (blkaddr < 0) 381 return; 382 383 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 384 nixlf, NIXLF_PROMISC_ENTRY); 385 386 entry.kw[0] = chan; 387 entry.kw_mask[0] = 0xFFFULL; 388 389 if (allmulti) { 390 kwi = NPC_PARSE_RESULT_DMAC_OFFSET / sizeof(u64); 391 entry.kw[kwi] = BIT_ULL(40); /* LSB bit of 1st byte in DMAC */ 392 entry.kw_mask[kwi] = BIT_ULL(40); 393 } 394 395 *(u64 *)&action = 0x00; 396 action.op = NIX_RX_ACTIONOP_UCAST; 397 action.pf_func = pcifunc; 398 399 entry.action = *(u64 *)&action; 400 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 401 NIX_INTF_RX, &entry, true); 402 } 403 404 static void npc_enadis_promisc_entry(struct rvu *rvu, u16 pcifunc, 405 int nixlf, bool enable) 406 { 407 struct npc_mcam *mcam = &rvu->hw->mcam; 408 int blkaddr, index; 409 410 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 411 if (blkaddr < 0) 412 return; 413 414 /* Only PF's have a promiscuous entry */ 415 if (pcifunc & RVU_PFVF_FUNC_MASK) 416 return; 417 418 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 419 nixlf, NIXLF_PROMISC_ENTRY); 420 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 421 } 422 423 void rvu_npc_disable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 424 { 425 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, false); 426 } 427 428 void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc, int nixlf) 429 { 430 npc_enadis_promisc_entry(rvu, pcifunc, nixlf, true); 431 } 432 433 void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, 434 int nixlf, u64 chan) 435 { 436 struct npc_mcam *mcam = &rvu->hw->mcam; 437 struct mcam_entry entry = { {0} }; 438 struct nix_rx_action action; 439 #ifdef MCAST_MCE 440 struct rvu_pfvf *pfvf; 441 #endif 442 int blkaddr, index; 443 444 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 445 if (blkaddr < 0) 446 return; 447 448 /* Only PF can add a bcast match entry */ 449 if (pcifunc & RVU_PFVF_FUNC_MASK) 450 return; 451 #ifdef MCAST_MCE 452 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 453 #endif 454 455 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 456 nixlf, NIXLF_BCAST_ENTRY); 457 458 /* Check for L2B bit and LMAC channel 459 * NOTE: Since MKEX default profile(a reduced version intended to 460 * accommodate more capability but igoring few bits) a stap-gap 461 * approach. 462 * Since we care for L2B which by HRM NPC_PARSE_KEX_S at BIT_POS[25], So 463 * moved to BIT_POS[13], ignoring ERRCODE, ERRLEV as we'll loose out 464 * on capability features needed for CoS (/from ODP PoV) e.g: VLAN, 465 * DSCP. 466 * 467 * Reduced layout of MKEX default profile - 468 * Includes following are (i.e.CHAN, L2/3{B/M}, LA, LB, LC, LD): 469 * 470 * BIT_POS[31:28] : LD 471 * BIT_POS[27:24] : LC 472 * BIT_POS[23:20] : LB 473 * BIT_POS[19:16] : LA 474 * BIT_POS[15:12] : L3B, L3M, L2B, L2M 475 * BIT_POS[11:00] : CHAN 476 * 477 */ 478 entry.kw[0] = BIT_ULL(13) | chan; 479 entry.kw_mask[0] = ~entry.kw[0] & (BIT_ULL(13) | 0xFFFULL); 480 481 *(u64 *)&action = 0x00; 482 #ifdef MCAST_MCE 483 /* Early silicon doesn't support pkt replication, 484 * so install entry with UCAST action, so that PF 485 * receives all broadcast packets. 486 */ 487 action.op = NIX_RX_ACTIONOP_MCAST; 488 action.pf_func = pcifunc; 489 action.index = pfvf->bcast_mce_idx; 490 #else 491 action.op = NIX_RX_ACTIONOP_UCAST; 492 action.pf_func = pcifunc; 493 #endif 494 495 entry.action = *(u64 *)&action; 496 npc_config_mcam_entry(rvu, mcam, blkaddr, index, 497 NIX_INTF_RX, &entry, true); 498 } 499 500 void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, 501 int group, int alg_idx, int mcam_index) 502 { 503 struct npc_mcam *mcam = &rvu->hw->mcam; 504 struct nix_rx_action action; 505 int blkaddr, index, bank; 506 507 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 508 if (blkaddr < 0) 509 return; 510 511 /* Check if this is for reserved default entry */ 512 if (mcam_index < 0) { 513 if (group != DEFAULT_RSS_CONTEXT_GROUP) 514 return; 515 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 516 nixlf, NIXLF_UCAST_ENTRY); 517 } else { 518 /* TODO: validate this mcam index */ 519 index = mcam_index; 520 } 521 522 if (index >= mcam->total_entries) 523 return; 524 525 bank = npc_get_bank(mcam, index); 526 index &= (mcam->banksize - 1); 527 528 *(u64 *)&action = rvu_read64(rvu, blkaddr, 529 NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); 530 /* Ignore if no action was set earlier */ 531 if (!*(u64 *)&action) 532 return; 533 534 action.op = NIX_RX_ACTIONOP_RSS; 535 action.pf_func = pcifunc; 536 action.index = group; 537 action.flow_key_alg = alg_idx; 538 539 rvu_write64(rvu, blkaddr, 540 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); 541 542 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 543 } 544 545 static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc, 546 int nixlf, bool enable) 547 { 548 struct npc_mcam *mcam = &rvu->hw->mcam; 549 struct nix_rx_action action; 550 int index, bank, blkaddr; 551 552 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 553 if (blkaddr < 0) 554 return; 555 556 /* Ucast MCAM match entry of this PF/VF */ 557 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 558 nixlf, NIXLF_UCAST_ENTRY); 559 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable); 560 561 /* For PF, ena/dis promisc and bcast MCAM match entries */ 562 if (pcifunc & RVU_PFVF_FUNC_MASK) 563 return; 564 565 /* For bcast, enable/disable only if it's action is not 566 * packet replication, incase if action is replication 567 * then this PF's nixlf is removed from bcast replication 568 * list. 569 */ 570 index = npc_get_nixlf_mcam_index(mcam, pcifunc, 571 nixlf, NIXLF_BCAST_ENTRY); 572 bank = npc_get_bank(mcam, index); 573 *(u64 *)&action = rvu_read64(rvu, blkaddr, 574 NPC_AF_MCAMEX_BANKX_ACTION(index & (mcam->banksize - 1), bank)); 575 if (action.op != NIX_RX_ACTIONOP_MCAST) 576 npc_enable_mcam_entry(rvu, mcam, 577 blkaddr, index, enable); 578 if (enable) 579 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf); 580 else 581 rvu_npc_disable_promisc_entry(rvu, pcifunc, nixlf); 582 583 rvu_npc_update_rxvlan(rvu, pcifunc, nixlf); 584 } 585 586 void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 587 { 588 npc_enadis_default_entries(rvu, pcifunc, nixlf, false); 589 } 590 591 void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 592 { 593 npc_enadis_default_entries(rvu, pcifunc, nixlf, true); 594 } 595 596 void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf) 597 { 598 struct npc_mcam *mcam = &rvu->hw->mcam; 599 int blkaddr; 600 601 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 602 if (blkaddr < 0) 603 return; 604 605 mutex_lock(&mcam->lock); 606 607 /* Disable and free all MCAM entries mapped to this 'pcifunc' */ 608 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 609 610 /* Free all MCAM counters mapped to this 'pcifunc' */ 611 npc_mcam_free_all_counters(rvu, mcam, pcifunc); 612 613 mutex_unlock(&mcam->lock); 614 615 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 616 } 617 618 #define SET_KEX_LD(intf, lid, ltype, ld, cfg) \ 619 rvu_write64(rvu, blkaddr, \ 620 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, ltype, ld), cfg) 621 622 #define SET_KEX_LDFLAGS(intf, ld, flags, cfg) \ 623 rvu_write64(rvu, blkaddr, \ 624 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, flags), cfg) 625 626 #define KEX_LD_CFG(bytesm1, hdr_ofs, ena, flags_ena, key_ofs) \ 627 (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ 628 ((flags_ena) << 6) | ((key_ofs) & 0x3F)) 629 630 static void npc_config_ldata_extract(struct rvu *rvu, int blkaddr) 631 { 632 struct npc_mcam *mcam = &rvu->hw->mcam; 633 int lid, ltype; 634 int lid_count; 635 u64 cfg; 636 637 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 638 lid_count = (cfg >> 4) & 0xF; 639 640 /* First clear any existing config i.e 641 * disable LDATA and FLAGS extraction. 642 */ 643 for (lid = 0; lid < lid_count; lid++) { 644 for (ltype = 0; ltype < 16; ltype++) { 645 SET_KEX_LD(NIX_INTF_RX, lid, ltype, 0, 0ULL); 646 SET_KEX_LD(NIX_INTF_RX, lid, ltype, 1, 0ULL); 647 SET_KEX_LD(NIX_INTF_TX, lid, ltype, 0, 0ULL); 648 SET_KEX_LD(NIX_INTF_TX, lid, ltype, 1, 0ULL); 649 650 SET_KEX_LDFLAGS(NIX_INTF_RX, 0, ltype, 0ULL); 651 SET_KEX_LDFLAGS(NIX_INTF_RX, 1, ltype, 0ULL); 652 SET_KEX_LDFLAGS(NIX_INTF_TX, 0, ltype, 0ULL); 653 SET_KEX_LDFLAGS(NIX_INTF_TX, 1, ltype, 0ULL); 654 } 655 } 656 657 if (mcam->keysize != NPC_MCAM_KEY_X2) 658 return; 659 660 /* Default MCAM KEX profile */ 661 /* Layer A: Ethernet: */ 662 663 /* DMAC: 6 bytes, KW1[47:0] */ 664 cfg = KEX_LD_CFG(0x05, 0x0, 0x1, 0x0, NPC_PARSE_RESULT_DMAC_OFFSET); 665 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 0, cfg); 666 667 /* Ethertype: 2 bytes, KW0[47:32] */ 668 cfg = KEX_LD_CFG(0x01, 0xc, 0x1, 0x0, 0x4); 669 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LA, NPC_LT_LA_ETHER, 1, cfg); 670 671 /* Layer B: Single VLAN (CTAG) */ 672 /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ 673 cfg = KEX_LD_CFG(0x03, 0x0, 0x1, 0x0, 0x4); 674 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_CTAG, 0, cfg); 675 676 /* Layer B: Stacked VLAN (STAG|QinQ) */ 677 /* CTAG VLAN[2..3] + Ethertype, 4 bytes, KW0[63:32] */ 678 cfg = KEX_LD_CFG(0x03, 0x4, 0x1, 0x0, 0x4); 679 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_STAG, 0, cfg); 680 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LB, NPC_LT_LB_QINQ, 0, cfg); 681 682 /* Layer C: IPv4 */ 683 /* SIP+DIP: 8 bytes, KW2[63:0] */ 684 cfg = KEX_LD_CFG(0x07, 0xc, 0x1, 0x0, 0x10); 685 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 0, cfg); 686 /* TOS: 1 byte, KW1[63:56] */ 687 cfg = KEX_LD_CFG(0x0, 0x1, 0x1, 0x0, 0xf); 688 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LC, NPC_LT_LC_IP, 1, cfg); 689 690 /* Layer D:UDP */ 691 /* SPORT: 2 bytes, KW3[15:0] */ 692 cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); 693 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 0, cfg); 694 /* DPORT: 2 bytes, KW3[31:16] */ 695 cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); 696 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_UDP, 1, cfg); 697 698 /* Layer D:TCP */ 699 /* SPORT: 2 bytes, KW3[15:0] */ 700 cfg = KEX_LD_CFG(0x1, 0x0, 0x1, 0x0, 0x18); 701 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 0, cfg); 702 /* DPORT: 2 bytes, KW3[31:16] */ 703 cfg = KEX_LD_CFG(0x1, 0x2, 0x1, 0x0, 0x1a); 704 SET_KEX_LD(NIX_INTF_RX, NPC_LID_LD, NPC_LT_LD_TCP, 1, cfg); 705 } 706 707 static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, 708 struct npc_kpu_profile_action *kpuaction, 709 int kpu, int entry, bool pkind) 710 { 711 struct npc_kpu_action0 action0 = {0}; 712 struct npc_kpu_action1 action1 = {0}; 713 u64 reg; 714 715 action1.errlev = kpuaction->errlev; 716 action1.errcode = kpuaction->errcode; 717 action1.dp0_offset = kpuaction->dp0_offset; 718 action1.dp1_offset = kpuaction->dp1_offset; 719 action1.dp2_offset = kpuaction->dp2_offset; 720 721 if (pkind) 722 reg = NPC_AF_PKINDX_ACTION1(entry); 723 else 724 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry); 725 726 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); 727 728 action0.byp_count = kpuaction->bypass_count; 729 action0.capture_ena = kpuaction->cap_ena; 730 action0.parse_done = kpuaction->parse_done; 731 action0.next_state = kpuaction->next_state; 732 action0.capture_lid = kpuaction->lid; 733 action0.capture_ltype = kpuaction->ltype; 734 action0.capture_flags = kpuaction->flags; 735 action0.ptr_advance = kpuaction->ptr_advance; 736 action0.var_len_offset = kpuaction->offset; 737 action0.var_len_mask = kpuaction->mask; 738 action0.var_len_right = kpuaction->right; 739 action0.var_len_shift = kpuaction->shift; 740 741 if (pkind) 742 reg = NPC_AF_PKINDX_ACTION0(entry); 743 else 744 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry); 745 746 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); 747 } 748 749 static void npc_config_kpucam(struct rvu *rvu, int blkaddr, 750 struct npc_kpu_profile_cam *kpucam, 751 int kpu, int entry) 752 { 753 struct npc_kpu_cam cam0 = {0}; 754 struct npc_kpu_cam cam1 = {0}; 755 756 cam1.state = kpucam->state & kpucam->state_mask; 757 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; 758 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; 759 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; 760 761 cam0.state = ~kpucam->state & kpucam->state_mask; 762 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; 763 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; 764 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; 765 766 rvu_write64(rvu, blkaddr, 767 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0); 768 rvu_write64(rvu, blkaddr, 769 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); 770 } 771 772 static inline u64 enable_mask(int count) 773 { 774 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); 775 } 776 777 static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, 778 struct npc_kpu_profile *profile) 779 { 780 int entry, num_entries, max_entries; 781 782 if (profile->cam_entries != profile->action_entries) { 783 dev_err(rvu->dev, 784 "KPU%d: CAM and action entries [%d != %d] not equal\n", 785 kpu, profile->cam_entries, profile->action_entries); 786 } 787 788 max_entries = rvu_read64(rvu, blkaddr, NPC_AF_CONST1) & 0xFFF; 789 790 /* Program CAM match entries for previous KPU extracted data */ 791 num_entries = min_t(int, profile->cam_entries, max_entries); 792 for (entry = 0; entry < num_entries; entry++) 793 npc_config_kpucam(rvu, blkaddr, 794 &profile->cam[entry], kpu, entry); 795 796 /* Program this KPU's actions */ 797 num_entries = min_t(int, profile->action_entries, max_entries); 798 for (entry = 0; entry < num_entries; entry++) 799 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry], 800 kpu, entry, false); 801 802 /* Enable all programmed entries */ 803 num_entries = min_t(int, profile->action_entries, profile->cam_entries); 804 rvu_write64(rvu, blkaddr, 805 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), enable_mask(num_entries)); 806 if (num_entries > 64) { 807 rvu_write64(rvu, blkaddr, 808 NPC_AF_KPUX_ENTRY_DISX(kpu, 1), 809 enable_mask(num_entries - 64)); 810 } 811 812 /* Enable this KPU */ 813 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); 814 } 815 816 static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) 817 { 818 struct rvu_hwinfo *hw = rvu->hw; 819 int num_pkinds, num_kpus, idx; 820 struct npc_pkind *pkind; 821 822 /* Get HW limits */ 823 hw->npc_kpus = (rvu_read64(rvu, blkaddr, NPC_AF_CONST) >> 8) & 0x1F; 824 825 /* Disable all KPUs and their entries */ 826 for (idx = 0; idx < hw->npc_kpus; idx++) { 827 rvu_write64(rvu, blkaddr, 828 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL); 829 rvu_write64(rvu, blkaddr, 830 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL); 831 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); 832 } 833 834 /* First program IKPU profile i.e PKIND configs. 835 * Check HW max count to avoid configuring junk or 836 * writing to unsupported CSR addresses. 837 */ 838 pkind = &hw->pkind; 839 num_pkinds = ARRAY_SIZE(ikpu_action_entries); 840 num_pkinds = min_t(int, pkind->rsrc.max, num_pkinds); 841 842 for (idx = 0; idx < num_pkinds; idx++) 843 npc_config_kpuaction(rvu, blkaddr, 844 &ikpu_action_entries[idx], 0, idx, true); 845 846 /* Program KPU CAM and Action profiles */ 847 num_kpus = ARRAY_SIZE(npc_kpu_profiles); 848 num_kpus = min_t(int, hw->npc_kpus, num_kpus); 849 850 for (idx = 0; idx < num_kpus; idx++) 851 npc_program_kpu_profile(rvu, blkaddr, 852 idx, &npc_kpu_profiles[idx]); 853 } 854 855 static int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) 856 { 857 int nixlf_count = rvu_get_nixlf_count(rvu); 858 struct npc_mcam *mcam = &rvu->hw->mcam; 859 int rsvd, err; 860 u64 cfg; 861 862 /* Get HW limits */ 863 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 864 mcam->banks = (cfg >> 44) & 0xF; 865 mcam->banksize = (cfg >> 28) & 0xFFFF; 866 mcam->counters.max = (cfg >> 48) & 0xFFFF; 867 868 /* Actual number of MCAM entries vary by entry size */ 869 cfg = (rvu_read64(rvu, blkaddr, 870 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; 871 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; 872 mcam->keysize = cfg; 873 874 /* Number of banks combined per MCAM entry */ 875 if (cfg == NPC_MCAM_KEY_X4) 876 mcam->banks_per_entry = 4; 877 else if (cfg == NPC_MCAM_KEY_X2) 878 mcam->banks_per_entry = 2; 879 else 880 mcam->banks_per_entry = 1; 881 882 /* Reserve one MCAM entry for each of the NIX LF to 883 * guarantee space to install default matching DMAC rule. 884 * Also reserve 2 MCAM entries for each PF for default 885 * channel based matching or 'bcast & promisc' matching to 886 * support BCAST and PROMISC modes of operation for PFs. 887 * PF0 is excluded. 888 */ 889 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + 890 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); 891 if (mcam->total_entries <= rsvd) { 892 dev_warn(rvu->dev, 893 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", 894 mcam->total_entries); 895 return -ENOMEM; 896 } 897 898 mcam->bmap_entries = mcam->total_entries - rsvd; 899 mcam->nixlf_offset = mcam->bmap_entries; 900 mcam->pf_offset = mcam->nixlf_offset + nixlf_count; 901 902 /* Allocate bitmaps for managing MCAM entries */ 903 mcam->bmap = devm_kcalloc(rvu->dev, BITS_TO_LONGS(mcam->bmap_entries), 904 sizeof(long), GFP_KERNEL); 905 if (!mcam->bmap) 906 return -ENOMEM; 907 908 mcam->bmap_reverse = devm_kcalloc(rvu->dev, 909 BITS_TO_LONGS(mcam->bmap_entries), 910 sizeof(long), GFP_KERNEL); 911 if (!mcam->bmap_reverse) 912 return -ENOMEM; 913 914 mcam->bmap_fcnt = mcam->bmap_entries; 915 916 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 917 mcam->entry2pfvf_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 918 sizeof(u16), GFP_KERNEL); 919 if (!mcam->entry2pfvf_map) 920 return -ENOMEM; 921 922 /* Reserve 1/8th of MCAM entries at the bottom for low priority 923 * allocations and another 1/8th at the top for high priority 924 * allocations. 925 */ 926 mcam->lprio_count = mcam->bmap_entries / 8; 927 if (mcam->lprio_count > BITS_PER_LONG) 928 mcam->lprio_count = round_down(mcam->lprio_count, 929 BITS_PER_LONG); 930 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; 931 mcam->hprio_count = mcam->lprio_count; 932 mcam->hprio_end = mcam->hprio_count; 933 934 /* Allocate bitmap for managing MCAM counters and memory 935 * for saving counter to RVU PFFUNC allocation mapping. 936 */ 937 err = rvu_alloc_bitmap(&mcam->counters); 938 if (err) 939 return err; 940 941 mcam->cntr2pfvf_map = devm_kcalloc(rvu->dev, mcam->counters.max, 942 sizeof(u16), GFP_KERNEL); 943 if (!mcam->cntr2pfvf_map) 944 goto free_mem; 945 946 /* Alloc memory for MCAM entry to counter mapping and for tracking 947 * counter's reference count. 948 */ 949 mcam->entry2cntr_map = devm_kcalloc(rvu->dev, mcam->bmap_entries, 950 sizeof(u16), GFP_KERNEL); 951 if (!mcam->entry2cntr_map) 952 goto free_mem; 953 954 mcam->cntr_refcnt = devm_kcalloc(rvu->dev, mcam->counters.max, 955 sizeof(u16), GFP_KERNEL); 956 if (!mcam->cntr_refcnt) 957 goto free_mem; 958 959 mutex_init(&mcam->lock); 960 961 return 0; 962 963 free_mem: 964 kfree(mcam->counters.bmap); 965 return -ENOMEM; 966 } 967 968 int rvu_npc_init(struct rvu *rvu) 969 { 970 struct npc_pkind *pkind = &rvu->hw->pkind; 971 u64 keyz = NPC_MCAM_KEY_X2; 972 int blkaddr, entry, bank, err; 973 u64 cfg, nibble_ena; 974 975 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 976 if (blkaddr < 0) { 977 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); 978 return -ENODEV; 979 } 980 981 /* First disable all MCAM entries, to stop traffic towards NIXLFs */ 982 cfg = rvu_read64(rvu, blkaddr, NPC_AF_CONST); 983 for (bank = 0; bank < ((cfg >> 44) & 0xF); bank++) { 984 for (entry = 0; entry < ((cfg >> 28) & 0xFFFF); entry++) 985 rvu_write64(rvu, blkaddr, 986 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0); 987 } 988 989 /* Allocate resource bimap for pkind*/ 990 pkind->rsrc.max = (rvu_read64(rvu, blkaddr, 991 NPC_AF_CONST1) >> 12) & 0xFF; 992 err = rvu_alloc_bitmap(&pkind->rsrc); 993 if (err) 994 return err; 995 996 /* Allocate mem for pkind to PF and channel mapping info */ 997 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max, 998 sizeof(u32), GFP_KERNEL); 999 if (!pkind->pfchan_map) 1000 return -ENOMEM; 1001 1002 /* Configure KPU profile */ 1003 npc_parser_profile_init(rvu, blkaddr); 1004 1005 /* Config Outer L2, IPv4's NPC layer info */ 1006 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, 1007 (NPC_LID_LA << 8) | (NPC_LT_LA_ETHER << 4) | 0x0F); 1008 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4, 1009 (NPC_LID_LC << 8) | (NPC_LT_LC_IP << 4) | 0x0F); 1010 1011 /* Enable below for Rx pkts. 1012 * - Outer IPv4 header checksum validation. 1013 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2M]. 1014 */ 1015 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG, 1016 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) | 1017 BIT_ULL(6) | BIT_ULL(2)); 1018 1019 /* Set RX and TX side MCAM search key size. 1020 * LA..LD (ltype only) + Channel 1021 */ 1022 nibble_ena = 0x49247; 1023 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX), 1024 ((keyz & 0x3) << 32) | nibble_ena); 1025 /* Due to an errata (35786) in A0 pass silicon, parse nibble enable 1026 * configuration has to be identical for both Rx and Tx interfaces. 1027 */ 1028 if (!is_rvu_9xxx_A0(rvu)) 1029 nibble_ena = (1ULL << 19) - 1; 1030 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX), 1031 ((keyz & 0x3) << 32) | nibble_ena); 1032 1033 err = npc_mcam_rsrcs_init(rvu, blkaddr); 1034 if (err) 1035 return err; 1036 1037 /* Config packet data and flags extraction into PARSE result */ 1038 npc_config_ldata_extract(rvu, blkaddr); 1039 1040 /* Set TX miss action to UCAST_DEFAULT i.e 1041 * transmit the packet on NIX LF SQ's default channel. 1042 */ 1043 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_TX), 1044 NIX_TX_ACTIONOP_UCAST_DEFAULT); 1045 1046 /* If MCAM lookup doesn't result in a match, drop the received packet */ 1047 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_MISS_ACT(NIX_INTF_RX), 1048 NIX_RX_ACTIONOP_DROP); 1049 1050 return 0; 1051 } 1052 1053 void rvu_npc_freemem(struct rvu *rvu) 1054 { 1055 struct npc_pkind *pkind = &rvu->hw->pkind; 1056 struct npc_mcam *mcam = &rvu->hw->mcam; 1057 1058 kfree(pkind->rsrc.bmap); 1059 kfree(mcam->counters.bmap); 1060 mutex_destroy(&mcam->lock); 1061 } 1062 1063 static int npc_mcam_verify_entry(struct npc_mcam *mcam, 1064 u16 pcifunc, int entry) 1065 { 1066 /* Verify if entry is valid and if it is indeed 1067 * allocated to the requesting PFFUNC. 1068 */ 1069 if (entry >= mcam->bmap_entries) 1070 return NPC_MCAM_INVALID_REQ; 1071 1072 if (pcifunc != mcam->entry2pfvf_map[entry]) 1073 return NPC_MCAM_PERM_DENIED; 1074 1075 return 0; 1076 } 1077 1078 static int npc_mcam_verify_counter(struct npc_mcam *mcam, 1079 u16 pcifunc, int cntr) 1080 { 1081 /* Verify if counter is valid and if it is indeed 1082 * allocated to the requesting PFFUNC. 1083 */ 1084 if (cntr >= mcam->counters.max) 1085 return NPC_MCAM_INVALID_REQ; 1086 1087 if (pcifunc != mcam->cntr2pfvf_map[cntr]) 1088 return NPC_MCAM_PERM_DENIED; 1089 1090 return 0; 1091 } 1092 1093 static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, 1094 int blkaddr, u16 entry, u16 cntr) 1095 { 1096 u16 index = entry & (mcam->banksize - 1); 1097 u16 bank = npc_get_bank(mcam, entry); 1098 1099 /* Set mapping and increment counter's refcnt */ 1100 mcam->entry2cntr_map[entry] = cntr; 1101 mcam->cntr_refcnt[cntr]++; 1102 /* Enable stats */ 1103 rvu_write64(rvu, blkaddr, 1104 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 1105 BIT_ULL(9) | cntr); 1106 } 1107 1108 static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, 1109 struct npc_mcam *mcam, 1110 int blkaddr, u16 entry, u16 cntr) 1111 { 1112 u16 index = entry & (mcam->banksize - 1); 1113 u16 bank = npc_get_bank(mcam, entry); 1114 1115 /* Remove mapping and reduce counter's refcnt */ 1116 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; 1117 mcam->cntr_refcnt[cntr]--; 1118 /* Disable stats */ 1119 rvu_write64(rvu, blkaddr, 1120 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); 1121 } 1122 1123 /* Sets MCAM entry in bitmap as used. Update 1124 * reverse bitmap too. Should be called with 1125 * 'mcam->lock' held. 1126 */ 1127 static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) 1128 { 1129 u16 entry, rentry; 1130 1131 entry = index; 1132 rentry = mcam->bmap_entries - index - 1; 1133 1134 __set_bit(entry, mcam->bmap); 1135 __set_bit(rentry, mcam->bmap_reverse); 1136 mcam->bmap_fcnt--; 1137 } 1138 1139 /* Sets MCAM entry in bitmap as free. Update 1140 * reverse bitmap too. Should be called with 1141 * 'mcam->lock' held. 1142 */ 1143 static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) 1144 { 1145 u16 entry, rentry; 1146 1147 entry = index; 1148 rentry = mcam->bmap_entries - index - 1; 1149 1150 __clear_bit(entry, mcam->bmap); 1151 __clear_bit(rentry, mcam->bmap_reverse); 1152 mcam->bmap_fcnt++; 1153 } 1154 1155 static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, 1156 int blkaddr, u16 pcifunc) 1157 { 1158 u16 index, cntr; 1159 1160 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ 1161 for (index = 0; index < mcam->bmap_entries; index++) { 1162 if (mcam->entry2pfvf_map[index] == pcifunc) { 1163 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP; 1164 /* Free the entry in bitmap */ 1165 npc_mcam_clear_bit(mcam, index); 1166 /* Disable the entry */ 1167 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false); 1168 1169 /* Update entry2counter mapping */ 1170 cntr = mcam->entry2cntr_map[index]; 1171 if (cntr != NPC_MCAM_INVALID_MAP) 1172 npc_unmap_mcam_entry_and_cntr(rvu, mcam, 1173 blkaddr, index, 1174 cntr); 1175 } 1176 } 1177 } 1178 1179 static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam, 1180 u16 pcifunc) 1181 { 1182 u16 cntr; 1183 1184 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */ 1185 for (cntr = 0; cntr < mcam->counters.max; cntr++) { 1186 if (mcam->cntr2pfvf_map[cntr] == pcifunc) { 1187 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP; 1188 mcam->cntr_refcnt[cntr] = 0; 1189 rvu_free_rsrc(&mcam->counters, cntr); 1190 /* This API is expected to be called after freeing 1191 * MCAM entries, which inturn will remove 1192 * 'entry to counter' mapping. 1193 * No need to do it again. 1194 */ 1195 } 1196 } 1197 } 1198 1199 /* Find area of contiguous free entries of size 'nr'. 1200 * If not found return max contiguous free entries available. 1201 */ 1202 static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start, 1203 u16 nr, u16 *max_area) 1204 { 1205 u16 max_area_start = 0; 1206 u16 index, next, end; 1207 1208 *max_area = 0; 1209 1210 again: 1211 index = find_next_zero_bit(map, size, start); 1212 if (index >= size) 1213 return max_area_start; 1214 1215 end = ((index + nr) >= size) ? size : index + nr; 1216 next = find_next_bit(map, end, index); 1217 if (*max_area < (next - index)) { 1218 *max_area = next - index; 1219 max_area_start = index; 1220 } 1221 1222 if (next < end) { 1223 start = next + 1; 1224 goto again; 1225 } 1226 1227 return max_area_start; 1228 } 1229 1230 /* Find number of free MCAM entries available 1231 * within range i.e in between 'start' and 'end'. 1232 */ 1233 static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end) 1234 { 1235 u16 index, next; 1236 u16 fcnt = 0; 1237 1238 again: 1239 if (start >= end) 1240 return fcnt; 1241 1242 index = find_next_zero_bit(map, end, start); 1243 if (index >= end) 1244 return fcnt; 1245 1246 next = find_next_bit(map, end, index); 1247 if (next <= end) { 1248 fcnt += next - index; 1249 start = next + 1; 1250 goto again; 1251 } 1252 1253 fcnt += end - index; 1254 return fcnt; 1255 } 1256 1257 static void 1258 npc_get_mcam_search_range_priority(struct npc_mcam *mcam, 1259 struct npc_mcam_alloc_entry_req *req, 1260 u16 *start, u16 *end, bool *reverse) 1261 { 1262 u16 fcnt; 1263 1264 if (req->priority == NPC_MCAM_HIGHER_PRIO) 1265 goto hprio; 1266 1267 /* For a low priority entry allocation 1268 * - If reference entry is not in hprio zone then 1269 * search range: ref_entry to end. 1270 * - If reference entry is in hprio zone and if 1271 * request can be accomodated in non-hprio zone then 1272 * search range: 'start of middle zone' to 'end' 1273 * - else search in reverse, so that less number of hprio 1274 * zone entries are allocated. 1275 */ 1276 1277 *reverse = false; 1278 *start = req->ref_entry + 1; 1279 *end = mcam->bmap_entries; 1280 1281 if (req->ref_entry >= mcam->hprio_end) 1282 return; 1283 1284 fcnt = npc_mcam_get_free_count(mcam->bmap, 1285 mcam->hprio_end, mcam->bmap_entries); 1286 if (fcnt > req->count) 1287 *start = mcam->hprio_end; 1288 else 1289 *reverse = true; 1290 return; 1291 1292 hprio: 1293 /* For a high priority entry allocation, search is always 1294 * in reverse to preserve hprio zone entries. 1295 * - If reference entry is not in lprio zone then 1296 * search range: 0 to ref_entry. 1297 * - If reference entry is in lprio zone and if 1298 * request can be accomodated in middle zone then 1299 * search range: 'hprio_end' to 'lprio_start' 1300 */ 1301 1302 *reverse = true; 1303 *start = 0; 1304 *end = req->ref_entry; 1305 1306 if (req->ref_entry <= mcam->lprio_start) 1307 return; 1308 1309 fcnt = npc_mcam_get_free_count(mcam->bmap, 1310 mcam->hprio_end, mcam->lprio_start); 1311 if (fcnt < req->count) 1312 return; 1313 *start = mcam->hprio_end; 1314 *end = mcam->lprio_start; 1315 } 1316 1317 static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, 1318 struct npc_mcam_alloc_entry_req *req, 1319 struct npc_mcam_alloc_entry_rsp *rsp) 1320 { 1321 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; 1322 u16 fcnt, hp_fcnt, lp_fcnt; 1323 u16 start, end, index; 1324 int entry, next_start; 1325 bool reverse = false; 1326 unsigned long *bmap; 1327 u16 max_contig; 1328 1329 mutex_lock(&mcam->lock); 1330 1331 /* Check if there are any free entries */ 1332 if (!mcam->bmap_fcnt) { 1333 mutex_unlock(&mcam->lock); 1334 return NPC_MCAM_ALLOC_FAILED; 1335 } 1336 1337 /* MCAM entries are divided into high priority, middle and 1338 * low priority zones. Idea is to not allocate top and lower 1339 * most entries as much as possible, this is to increase 1340 * probability of honouring priority allocation requests. 1341 * 1342 * Two bitmaps are used for mcam entry management, 1343 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'. 1344 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'. 1345 * 1346 * Reverse bitmap is used to allocate entries 1347 * - when a higher priority entry is requested 1348 * - when available free entries are less. 1349 * Lower priority ones out of avaialble free entries are always 1350 * chosen when 'high vs low' question arises. 1351 */ 1352 1353 /* Get the search range for priority allocation request */ 1354 if (req->priority) { 1355 npc_get_mcam_search_range_priority(mcam, req, 1356 &start, &end, &reverse); 1357 goto alloc; 1358 } 1359 1360 /* Find out the search range for non-priority allocation request 1361 * 1362 * Get MCAM free entry count in middle zone. 1363 */ 1364 lp_fcnt = npc_mcam_get_free_count(mcam->bmap, 1365 mcam->lprio_start, 1366 mcam->bmap_entries); 1367 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end); 1368 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt; 1369 1370 /* Check if request can be accomodated in the middle zone */ 1371 if (fcnt > req->count) { 1372 start = mcam->hprio_end; 1373 end = mcam->lprio_start; 1374 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) { 1375 /* Expand search zone from half of hprio zone to 1376 * half of lprio zone. 1377 */ 1378 start = mcam->hprio_end / 2; 1379 end = mcam->bmap_entries - (mcam->lprio_count / 2); 1380 reverse = true; 1381 } else { 1382 /* Not enough free entries, search all entries in reverse, 1383 * so that low priority ones will get used up. 1384 */ 1385 reverse = true; 1386 start = 0; 1387 end = mcam->bmap_entries; 1388 } 1389 1390 alloc: 1391 if (reverse) { 1392 bmap = mcam->bmap_reverse; 1393 start = mcam->bmap_entries - start; 1394 end = mcam->bmap_entries - end; 1395 index = start; 1396 start = end; 1397 end = index; 1398 } else { 1399 bmap = mcam->bmap; 1400 } 1401 1402 if (req->contig) { 1403 /* Allocate requested number of contiguous entries, if 1404 * unsuccessful find max contiguous entries available. 1405 */ 1406 index = npc_mcam_find_zero_area(bmap, end, start, 1407 req->count, &max_contig); 1408 rsp->count = max_contig; 1409 if (reverse) 1410 rsp->entry = mcam->bmap_entries - index - max_contig; 1411 else 1412 rsp->entry = index; 1413 } else { 1414 /* Allocate requested number of non-contiguous entries, 1415 * if unsuccessful allocate as many as possible. 1416 */ 1417 rsp->count = 0; 1418 next_start = start; 1419 for (entry = 0; entry < req->count; entry++) { 1420 index = find_next_zero_bit(bmap, end, next_start); 1421 if (index >= end) 1422 break; 1423 1424 next_start = start + (index - start) + 1; 1425 1426 /* Save the entry's index */ 1427 if (reverse) 1428 index = mcam->bmap_entries - index - 1; 1429 entry_list[entry] = index; 1430 rsp->count++; 1431 } 1432 } 1433 1434 /* If allocating requested no of entries is unsucessful, 1435 * expand the search range to full bitmap length and retry. 1436 */ 1437 if (!req->priority && (rsp->count < req->count) && 1438 ((end - start) != mcam->bmap_entries)) { 1439 reverse = true; 1440 start = 0; 1441 end = mcam->bmap_entries; 1442 goto alloc; 1443 } 1444 1445 /* For priority entry allocation requests, if allocation is 1446 * failed then expand search to max possible range and retry. 1447 */ 1448 if (req->priority && rsp->count < req->count) { 1449 if (req->priority == NPC_MCAM_LOWER_PRIO && 1450 (start != (req->ref_entry + 1))) { 1451 start = req->ref_entry + 1; 1452 end = mcam->bmap_entries; 1453 reverse = false; 1454 goto alloc; 1455 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && 1456 ((end - start) != req->ref_entry)) { 1457 start = 0; 1458 end = req->ref_entry; 1459 reverse = true; 1460 goto alloc; 1461 } 1462 } 1463 1464 /* Copy MCAM entry indices into mbox response entry_list. 1465 * Requester always expects indices in ascending order, so 1466 * so reverse the list if reverse bitmap is used for allocation. 1467 */ 1468 if (!req->contig && rsp->count) { 1469 index = 0; 1470 for (entry = rsp->count - 1; entry >= 0; entry--) { 1471 if (reverse) 1472 rsp->entry_list[index++] = entry_list[entry]; 1473 else 1474 rsp->entry_list[entry] = entry_list[entry]; 1475 } 1476 } 1477 1478 /* Mark the allocated entries as used and set nixlf mapping */ 1479 for (entry = 0; entry < rsp->count; entry++) { 1480 index = req->contig ? 1481 (rsp->entry + entry) : rsp->entry_list[entry]; 1482 npc_mcam_set_bit(mcam, index); 1483 mcam->entry2pfvf_map[index] = pcifunc; 1484 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; 1485 } 1486 1487 /* Update available free count in mbox response */ 1488 rsp->free_count = mcam->bmap_fcnt; 1489 1490 mutex_unlock(&mcam->lock); 1491 return 0; 1492 } 1493 1494 int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, 1495 struct npc_mcam_alloc_entry_req *req, 1496 struct npc_mcam_alloc_entry_rsp *rsp) 1497 { 1498 struct npc_mcam *mcam = &rvu->hw->mcam; 1499 u16 pcifunc = req->hdr.pcifunc; 1500 int blkaddr; 1501 1502 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1503 if (blkaddr < 0) 1504 return NPC_MCAM_INVALID_REQ; 1505 1506 rsp->entry = NPC_MCAM_ENTRY_INVALID; 1507 rsp->free_count = 0; 1508 1509 /* Check if ref_entry is within range */ 1510 if (req->priority && req->ref_entry >= mcam->bmap_entries) 1511 return NPC_MCAM_INVALID_REQ; 1512 1513 /* ref_entry can't be '0' if requested priority is high. 1514 * Can't be last entry if requested priority is low. 1515 */ 1516 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || 1517 ((req->ref_entry == (mcam->bmap_entries - 1)) && 1518 req->priority == NPC_MCAM_LOWER_PRIO)) 1519 return NPC_MCAM_INVALID_REQ; 1520 1521 /* Since list of allocated indices needs to be sent to requester, 1522 * max number of non-contiguous entries per mbox msg is limited. 1523 */ 1524 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) 1525 return NPC_MCAM_INVALID_REQ; 1526 1527 /* Alloc request from PFFUNC with no NIXLF attached should be denied */ 1528 if (!is_nixlf_attached(rvu, pcifunc)) 1529 return NPC_MCAM_ALLOC_DENIED; 1530 1531 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp); 1532 } 1533 1534 int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, 1535 struct npc_mcam_free_entry_req *req, 1536 struct msg_rsp *rsp) 1537 { 1538 struct npc_mcam *mcam = &rvu->hw->mcam; 1539 u16 pcifunc = req->hdr.pcifunc; 1540 int blkaddr, rc = 0; 1541 u16 cntr; 1542 1543 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1544 if (blkaddr < 0) 1545 return NPC_MCAM_INVALID_REQ; 1546 1547 /* Free request from PFFUNC with no NIXLF attached, ignore */ 1548 if (!is_nixlf_attached(rvu, pcifunc)) 1549 return NPC_MCAM_INVALID_REQ; 1550 1551 mutex_lock(&mcam->lock); 1552 1553 if (req->all) 1554 goto free_all; 1555 1556 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1557 if (rc) 1558 goto exit; 1559 1560 mcam->entry2pfvf_map[req->entry] = 0; 1561 npc_mcam_clear_bit(mcam, req->entry); 1562 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1563 1564 /* Update entry2counter mapping */ 1565 cntr = mcam->entry2cntr_map[req->entry]; 1566 if (cntr != NPC_MCAM_INVALID_MAP) 1567 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1568 req->entry, cntr); 1569 1570 goto exit; 1571 1572 free_all: 1573 /* Free up all entries allocated to requesting PFFUNC */ 1574 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc); 1575 exit: 1576 mutex_unlock(&mcam->lock); 1577 return rc; 1578 } 1579 1580 int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, 1581 struct npc_mcam_write_entry_req *req, 1582 struct msg_rsp *rsp) 1583 { 1584 struct npc_mcam *mcam = &rvu->hw->mcam; 1585 u16 pcifunc = req->hdr.pcifunc; 1586 int blkaddr, rc; 1587 1588 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1589 if (blkaddr < 0) 1590 return NPC_MCAM_INVALID_REQ; 1591 1592 mutex_lock(&mcam->lock); 1593 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1594 if (rc) 1595 goto exit; 1596 1597 if (req->set_cntr && 1598 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { 1599 rc = NPC_MCAM_INVALID_REQ; 1600 goto exit; 1601 } 1602 1603 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) { 1604 rc = NPC_MCAM_INVALID_REQ; 1605 goto exit; 1606 } 1607 1608 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, req->intf, 1609 &req->entry_data, req->enable_entry); 1610 1611 if (req->set_cntr) 1612 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1613 req->entry, req->cntr); 1614 1615 rc = 0; 1616 exit: 1617 mutex_unlock(&mcam->lock); 1618 return rc; 1619 } 1620 1621 int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, 1622 struct npc_mcam_ena_dis_entry_req *req, 1623 struct msg_rsp *rsp) 1624 { 1625 struct npc_mcam *mcam = &rvu->hw->mcam; 1626 u16 pcifunc = req->hdr.pcifunc; 1627 int blkaddr, rc; 1628 1629 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1630 if (blkaddr < 0) 1631 return NPC_MCAM_INVALID_REQ; 1632 1633 mutex_lock(&mcam->lock); 1634 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1635 mutex_unlock(&mcam->lock); 1636 if (rc) 1637 return rc; 1638 1639 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true); 1640 1641 return 0; 1642 } 1643 1644 int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, 1645 struct npc_mcam_ena_dis_entry_req *req, 1646 struct msg_rsp *rsp) 1647 { 1648 struct npc_mcam *mcam = &rvu->hw->mcam; 1649 u16 pcifunc = req->hdr.pcifunc; 1650 int blkaddr, rc; 1651 1652 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1653 if (blkaddr < 0) 1654 return NPC_MCAM_INVALID_REQ; 1655 1656 mutex_lock(&mcam->lock); 1657 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); 1658 mutex_unlock(&mcam->lock); 1659 if (rc) 1660 return rc; 1661 1662 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false); 1663 1664 return 0; 1665 } 1666 1667 int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, 1668 struct npc_mcam_shift_entry_req *req, 1669 struct npc_mcam_shift_entry_rsp *rsp) 1670 { 1671 struct npc_mcam *mcam = &rvu->hw->mcam; 1672 u16 pcifunc = req->hdr.pcifunc; 1673 u16 old_entry, new_entry; 1674 u16 index, cntr; 1675 int blkaddr, rc; 1676 1677 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1678 if (blkaddr < 0) 1679 return NPC_MCAM_INVALID_REQ; 1680 1681 if (req->shift_count > NPC_MCAM_MAX_SHIFTS) 1682 return NPC_MCAM_INVALID_REQ; 1683 1684 mutex_lock(&mcam->lock); 1685 for (index = 0; index < req->shift_count; index++) { 1686 old_entry = req->curr_entry[index]; 1687 new_entry = req->new_entry[index]; 1688 1689 /* Check if both old and new entries are valid and 1690 * does belong to this PFFUNC or not. 1691 */ 1692 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry); 1693 if (rc) 1694 break; 1695 1696 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry); 1697 if (rc) 1698 break; 1699 1700 /* new_entry should not have a counter mapped */ 1701 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) { 1702 rc = NPC_MCAM_PERM_DENIED; 1703 break; 1704 } 1705 1706 /* Disable the new_entry */ 1707 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false); 1708 1709 /* Copy rule from old entry to new entry */ 1710 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry); 1711 1712 /* Copy counter mapping, if any */ 1713 cntr = mcam->entry2cntr_map[old_entry]; 1714 if (cntr != NPC_MCAM_INVALID_MAP) { 1715 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1716 old_entry, cntr); 1717 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1718 new_entry, cntr); 1719 } 1720 1721 /* Enable new_entry and disable old_entry */ 1722 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true); 1723 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false); 1724 } 1725 1726 /* If shift has failed then report the failed index */ 1727 if (index != req->shift_count) { 1728 rc = NPC_MCAM_PERM_DENIED; 1729 rsp->failed_entry_idx = index; 1730 } 1731 1732 mutex_unlock(&mcam->lock); 1733 return rc; 1734 } 1735 1736 int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, 1737 struct npc_mcam_alloc_counter_req *req, 1738 struct npc_mcam_alloc_counter_rsp *rsp) 1739 { 1740 struct npc_mcam *mcam = &rvu->hw->mcam; 1741 u16 pcifunc = req->hdr.pcifunc; 1742 u16 max_contig, cntr; 1743 int blkaddr, index; 1744 1745 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1746 if (blkaddr < 0) 1747 return NPC_MCAM_INVALID_REQ; 1748 1749 /* If the request is from a PFFUNC with no NIXLF attached, ignore */ 1750 if (!is_nixlf_attached(rvu, pcifunc)) 1751 return NPC_MCAM_INVALID_REQ; 1752 1753 /* Since list of allocated counter IDs needs to be sent to requester, 1754 * max number of non-contiguous counters per mbox msg is limited. 1755 */ 1756 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) 1757 return NPC_MCAM_INVALID_REQ; 1758 1759 mutex_lock(&mcam->lock); 1760 1761 /* Check if unused counters are available or not */ 1762 if (!rvu_rsrc_free_count(&mcam->counters)) { 1763 mutex_unlock(&mcam->lock); 1764 return NPC_MCAM_ALLOC_FAILED; 1765 } 1766 1767 rsp->count = 0; 1768 1769 if (req->contig) { 1770 /* Allocate requested number of contiguous counters, if 1771 * unsuccessful find max contiguous entries available. 1772 */ 1773 index = npc_mcam_find_zero_area(mcam->counters.bmap, 1774 mcam->counters.max, 0, 1775 req->count, &max_contig); 1776 rsp->count = max_contig; 1777 rsp->cntr = index; 1778 for (cntr = index; cntr < (index + max_contig); cntr++) { 1779 __set_bit(cntr, mcam->counters.bmap); 1780 mcam->cntr2pfvf_map[cntr] = pcifunc; 1781 } 1782 } else { 1783 /* Allocate requested number of non-contiguous counters, 1784 * if unsuccessful allocate as many as possible. 1785 */ 1786 for (cntr = 0; cntr < req->count; cntr++) { 1787 index = rvu_alloc_rsrc(&mcam->counters); 1788 if (index < 0) 1789 break; 1790 rsp->cntr_list[cntr] = index; 1791 rsp->count++; 1792 mcam->cntr2pfvf_map[index] = pcifunc; 1793 } 1794 } 1795 1796 mutex_unlock(&mcam->lock); 1797 return 0; 1798 } 1799 1800 int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, 1801 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 1802 { 1803 struct npc_mcam *mcam = &rvu->hw->mcam; 1804 u16 index, entry = 0; 1805 int blkaddr, err; 1806 1807 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1808 if (blkaddr < 0) 1809 return NPC_MCAM_INVALID_REQ; 1810 1811 mutex_lock(&mcam->lock); 1812 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1813 if (err) { 1814 mutex_unlock(&mcam->lock); 1815 return err; 1816 } 1817 1818 /* Mark counter as free/unused */ 1819 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP; 1820 rvu_free_rsrc(&mcam->counters, req->cntr); 1821 1822 /* Disable all MCAM entry's stats which are using this counter */ 1823 while (entry < mcam->bmap_entries) { 1824 if (!mcam->cntr_refcnt[req->cntr]) 1825 break; 1826 1827 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 1828 if (index >= mcam->bmap_entries) 1829 break; 1830 if (mcam->entry2cntr_map[index] != req->cntr) 1831 continue; 1832 1833 entry = index + 1; 1834 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1835 index, req->cntr); 1836 } 1837 1838 mutex_unlock(&mcam->lock); 1839 return 0; 1840 } 1841 1842 int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, 1843 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp) 1844 { 1845 struct npc_mcam *mcam = &rvu->hw->mcam; 1846 u16 index, entry = 0; 1847 int blkaddr, rc; 1848 1849 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1850 if (blkaddr < 0) 1851 return NPC_MCAM_INVALID_REQ; 1852 1853 mutex_lock(&mcam->lock); 1854 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1855 if (rc) 1856 goto exit; 1857 1858 /* Unmap the MCAM entry and counter */ 1859 if (!req->all) { 1860 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry); 1861 if (rc) 1862 goto exit; 1863 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1864 req->entry, req->cntr); 1865 goto exit; 1866 } 1867 1868 /* Disable all MCAM entry's stats which are using this counter */ 1869 while (entry < mcam->bmap_entries) { 1870 if (!mcam->cntr_refcnt[req->cntr]) 1871 break; 1872 1873 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry); 1874 if (index >= mcam->bmap_entries) 1875 break; 1876 if (mcam->entry2cntr_map[index] != req->cntr) 1877 continue; 1878 1879 entry = index + 1; 1880 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, 1881 index, req->cntr); 1882 } 1883 exit: 1884 mutex_unlock(&mcam->lock); 1885 return rc; 1886 } 1887 1888 int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, 1889 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) 1890 { 1891 struct npc_mcam *mcam = &rvu->hw->mcam; 1892 int blkaddr, err; 1893 1894 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1895 if (blkaddr < 0) 1896 return NPC_MCAM_INVALID_REQ; 1897 1898 mutex_lock(&mcam->lock); 1899 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1900 mutex_unlock(&mcam->lock); 1901 if (err) 1902 return err; 1903 1904 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00); 1905 1906 return 0; 1907 } 1908 1909 int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, 1910 struct npc_mcam_oper_counter_req *req, 1911 struct npc_mcam_oper_counter_rsp *rsp) 1912 { 1913 struct npc_mcam *mcam = &rvu->hw->mcam; 1914 int blkaddr, err; 1915 1916 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1917 if (blkaddr < 0) 1918 return NPC_MCAM_INVALID_REQ; 1919 1920 mutex_lock(&mcam->lock); 1921 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); 1922 mutex_unlock(&mcam->lock); 1923 if (err) 1924 return err; 1925 1926 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr)); 1927 rsp->stat &= BIT_ULL(48) - 1; 1928 1929 return 0; 1930 } 1931 1932 int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, 1933 struct npc_mcam_alloc_and_write_entry_req *req, 1934 struct npc_mcam_alloc_and_write_entry_rsp *rsp) 1935 { 1936 struct npc_mcam_alloc_counter_req cntr_req; 1937 struct npc_mcam_alloc_counter_rsp cntr_rsp; 1938 struct npc_mcam_alloc_entry_req entry_req; 1939 struct npc_mcam_alloc_entry_rsp entry_rsp; 1940 struct npc_mcam *mcam = &rvu->hw->mcam; 1941 u16 entry = NPC_MCAM_ENTRY_INVALID; 1942 u16 cntr = NPC_MCAM_ENTRY_INVALID; 1943 int blkaddr, rc; 1944 1945 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 1946 if (blkaddr < 0) 1947 return NPC_MCAM_INVALID_REQ; 1948 1949 if (req->intf != NIX_INTF_RX && req->intf != NIX_INTF_TX) 1950 return NPC_MCAM_INVALID_REQ; 1951 1952 /* Try to allocate a MCAM entry */ 1953 entry_req.hdr.pcifunc = req->hdr.pcifunc; 1954 entry_req.contig = true; 1955 entry_req.priority = req->priority; 1956 entry_req.ref_entry = req->ref_entry; 1957 entry_req.count = 1; 1958 1959 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, 1960 &entry_req, &entry_rsp); 1961 if (rc) 1962 return rc; 1963 1964 if (!entry_rsp.count) 1965 return NPC_MCAM_ALLOC_FAILED; 1966 1967 entry = entry_rsp.entry; 1968 1969 if (!req->alloc_cntr) 1970 goto write_entry; 1971 1972 /* Now allocate counter */ 1973 cntr_req.hdr.pcifunc = req->hdr.pcifunc; 1974 cntr_req.contig = true; 1975 cntr_req.count = 1; 1976 1977 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); 1978 if (rc) { 1979 /* Free allocated MCAM entry */ 1980 mutex_lock(&mcam->lock); 1981 mcam->entry2pfvf_map[entry] = 0; 1982 npc_mcam_clear_bit(mcam, entry); 1983 mutex_unlock(&mcam->lock); 1984 return rc; 1985 } 1986 1987 cntr = cntr_rsp.cntr; 1988 1989 write_entry: 1990 mutex_lock(&mcam->lock); 1991 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, req->intf, 1992 &req->entry_data, req->enable_entry); 1993 1994 if (req->alloc_cntr) 1995 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr); 1996 mutex_unlock(&mcam->lock); 1997 1998 rsp->entry = entry; 1999 rsp->cntr = cntr; 2000 2001 return 0; 2002 } 2003 2004 #define GET_KEX_CFG(intf) \ 2005 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) 2006 2007 #define GET_KEX_FLAGS(ld) \ 2008 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld)) 2009 2010 #define GET_KEX_LD(intf, lid, lt, ld) \ 2011 rvu_read64(rvu, BLKADDR_NPC, \ 2012 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld)) 2013 2014 #define GET_KEX_LDFLAGS(intf, ld, fl) \ 2015 rvu_read64(rvu, BLKADDR_NPC, \ 2016 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl)) 2017 2018 int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req, 2019 struct npc_get_kex_cfg_rsp *rsp) 2020 { 2021 int lid, lt, ld, fl; 2022 2023 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX); 2024 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX); 2025 for (lid = 0; lid < NPC_MAX_LID; lid++) { 2026 for (lt = 0; lt < NPC_MAX_LT; lt++) { 2027 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2028 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] = 2029 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld); 2030 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] = 2031 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld); 2032 } 2033 } 2034 } 2035 for (ld = 0; ld < NPC_MAX_LD; ld++) 2036 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld); 2037 2038 for (ld = 0; ld < NPC_MAX_LD; ld++) { 2039 for (fl = 0; fl < NPC_MAX_LFL; fl++) { 2040 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] = 2041 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl); 2042 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] = 2043 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl); 2044 } 2045 } 2046 return 0; 2047 } 2048 2049 int rvu_npc_update_rxvlan(struct rvu *rvu, u16 pcifunc, int nixlf) 2050 { 2051 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 2052 struct npc_mcam *mcam = &rvu->hw->mcam; 2053 int blkaddr, index; 2054 bool enable; 2055 2056 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2057 if (blkaddr < 0) 2058 return NIX_AF_ERR_AF_LF_INVALID; 2059 2060 if (!pfvf->rxvlan) 2061 return 0; 2062 2063 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, 2064 NIXLF_UCAST_ENTRY); 2065 pfvf->entry.action = npc_get_mcam_action(rvu, mcam, blkaddr, index); 2066 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, index); 2067 npc_config_mcam_entry(rvu, mcam, blkaddr, pfvf->rxvlan_index, 2068 NIX_INTF_RX, &pfvf->entry, enable); 2069 2070 return 0; 2071 } 2072