1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/module.h> 9 #include <linux/pci.h> 10 11 #include "rvu_struct.h" 12 #include "rvu_reg.h" 13 #include "rvu.h" 14 #include "npc.h" 15 #include "cgx.h" 16 #include "lmac_common.h" 17 #include "rvu_npc_hash.h" 18 19 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc); 20 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 21 int type, int chan_id); 22 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 23 int type, bool add); 24 static int nix_setup_ipolicers(struct rvu *rvu, 25 struct nix_hw *nix_hw, int blkaddr); 26 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw); 27 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 28 struct nix_hw *nix_hw, u16 pcifunc); 29 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc); 30 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 31 u32 leaf_prof); 32 static const char *nix_get_ctx_name(int ctype); 33 34 enum mc_tbl_sz { 35 MC_TBL_SZ_256, 36 MC_TBL_SZ_512, 37 MC_TBL_SZ_1K, 38 MC_TBL_SZ_2K, 39 MC_TBL_SZ_4K, 40 MC_TBL_SZ_8K, 41 MC_TBL_SZ_16K, 42 MC_TBL_SZ_32K, 43 MC_TBL_SZ_64K, 44 }; 45 46 enum mc_buf_cnt { 47 MC_BUF_CNT_8, 48 MC_BUF_CNT_16, 49 MC_BUF_CNT_32, 50 MC_BUF_CNT_64, 51 MC_BUF_CNT_128, 52 MC_BUF_CNT_256, 53 MC_BUF_CNT_512, 54 MC_BUF_CNT_1024, 55 MC_BUF_CNT_2048, 56 }; 57 58 enum nix_makr_fmt_indexes { 59 NIX_MARK_CFG_IP_DSCP_RED, 60 NIX_MARK_CFG_IP_DSCP_YELLOW, 61 NIX_MARK_CFG_IP_DSCP_YELLOW_RED, 62 NIX_MARK_CFG_IP_ECN_RED, 63 NIX_MARK_CFG_IP_ECN_YELLOW, 64 NIX_MARK_CFG_IP_ECN_YELLOW_RED, 65 NIX_MARK_CFG_VLAN_DEI_RED, 66 NIX_MARK_CFG_VLAN_DEI_YELLOW, 67 NIX_MARK_CFG_VLAN_DEI_YELLOW_RED, 68 NIX_MARK_CFG_MAX, 69 }; 70 71 /* For now considering MC resources needed for broadcast 72 * pkt replication only. i.e 256 HWVFs + 12 PFs. 73 */ 74 #define MC_TBL_SIZE MC_TBL_SZ_512 75 #define MC_BUF_CNT MC_BUF_CNT_128 76 77 struct mce { 78 struct hlist_node node; 79 u16 pcifunc; 80 }; 81 82 int rvu_get_next_nix_blkaddr(struct rvu *rvu, int blkaddr) 83 { 84 int i = 0; 85 86 /*If blkaddr is 0, return the first nix block address*/ 87 if (blkaddr == 0) 88 return rvu->nix_blkaddr[blkaddr]; 89 90 while (i + 1 < MAX_NIX_BLKS) { 91 if (rvu->nix_blkaddr[i] == blkaddr) 92 return rvu->nix_blkaddr[i + 1]; 93 i++; 94 } 95 96 return 0; 97 } 98 99 bool is_nixlf_attached(struct rvu *rvu, u16 pcifunc) 100 { 101 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 102 int blkaddr; 103 104 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 105 if (!pfvf->nixlf || blkaddr < 0) 106 return false; 107 return true; 108 } 109 110 int rvu_get_nixlf_count(struct rvu *rvu) 111 { 112 int blkaddr = 0, max = 0; 113 struct rvu_block *block; 114 115 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 116 while (blkaddr) { 117 block = &rvu->hw->block[blkaddr]; 118 max += block->lf.max; 119 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 120 } 121 return max; 122 } 123 124 int nix_get_nixlf(struct rvu *rvu, u16 pcifunc, int *nixlf, int *nix_blkaddr) 125 { 126 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 127 struct rvu_hwinfo *hw = rvu->hw; 128 int blkaddr; 129 130 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 131 if (!pfvf->nixlf || blkaddr < 0) 132 return NIX_AF_ERR_AF_LF_INVALID; 133 134 *nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 135 if (*nixlf < 0) 136 return NIX_AF_ERR_AF_LF_INVALID; 137 138 if (nix_blkaddr) 139 *nix_blkaddr = blkaddr; 140 141 return 0; 142 } 143 144 int nix_get_struct_ptrs(struct rvu *rvu, u16 pcifunc, 145 struct nix_hw **nix_hw, int *blkaddr) 146 { 147 struct rvu_pfvf *pfvf; 148 149 pfvf = rvu_get_pfvf(rvu, pcifunc); 150 *blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 151 if (!pfvf->nixlf || *blkaddr < 0) 152 return NIX_AF_ERR_AF_LF_INVALID; 153 154 *nix_hw = get_nix_hw(rvu->hw, *blkaddr); 155 if (!*nix_hw) 156 return NIX_AF_ERR_INVALID_NIXBLK; 157 return 0; 158 } 159 160 static void nix_mce_list_init(struct nix_mce_list *list, int max) 161 { 162 INIT_HLIST_HEAD(&list->head); 163 list->count = 0; 164 list->max = max; 165 } 166 167 static u16 nix_alloc_mce_list(struct nix_mcast *mcast, int count) 168 { 169 int idx; 170 171 if (!mcast) 172 return 0; 173 174 idx = mcast->next_free_mce; 175 mcast->next_free_mce += count; 176 return idx; 177 } 178 179 struct nix_hw *get_nix_hw(struct rvu_hwinfo *hw, int blkaddr) 180 { 181 int nix_blkaddr = 0, i = 0; 182 struct rvu *rvu = hw->rvu; 183 184 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 185 while (nix_blkaddr) { 186 if (blkaddr == nix_blkaddr && hw->nix) 187 return &hw->nix[i]; 188 nix_blkaddr = rvu_get_next_nix_blkaddr(rvu, nix_blkaddr); 189 i++; 190 } 191 return NULL; 192 } 193 194 u32 convert_dwrr_mtu_to_bytes(u8 dwrr_mtu) 195 { 196 dwrr_mtu &= 0x1FULL; 197 198 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 199 * Value of 4 is reserved for MTU value of 9728 bytes. 200 * Value of 5 is reserved for MTU value of 10240 bytes. 201 */ 202 switch (dwrr_mtu) { 203 case 4: 204 return 9728; 205 case 5: 206 return 10240; 207 default: 208 return BIT_ULL(dwrr_mtu); 209 } 210 211 return 0; 212 } 213 214 u32 convert_bytes_to_dwrr_mtu(u32 bytes) 215 { 216 /* MTU used for DWRR calculation is in power of 2 up until 64K bytes. 217 * Value of 4 is reserved for MTU value of 9728 bytes. 218 * Value of 5 is reserved for MTU value of 10240 bytes. 219 */ 220 if (bytes > BIT_ULL(16)) 221 return 0; 222 223 switch (bytes) { 224 case 9728: 225 return 4; 226 case 10240: 227 return 5; 228 default: 229 return ilog2(bytes); 230 } 231 232 return 0; 233 } 234 235 static void nix_rx_sync(struct rvu *rvu, int blkaddr) 236 { 237 int err; 238 239 /* Sync all in flight RX packets to LLC/DRAM */ 240 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 241 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 242 if (err) 243 dev_err(rvu->dev, "SYNC1: NIX RX software sync failed\n"); 244 245 /* SW_SYNC ensures all existing transactions are finished and pkts 246 * are written to LLC/DRAM, queues should be teared down after 247 * successful SW_SYNC. Due to a HW errata, in some rare scenarios 248 * an existing transaction might end after SW_SYNC operation. To 249 * ensure operation is fully done, do the SW_SYNC twice. 250 */ 251 rvu_write64(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0)); 252 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_RX_SW_SYNC, BIT_ULL(0), true); 253 if (err) 254 dev_err(rvu->dev, "SYNC2: NIX RX software sync failed\n"); 255 } 256 257 static bool is_valid_txschq(struct rvu *rvu, int blkaddr, 258 int lvl, u16 pcifunc, u16 schq) 259 { 260 struct rvu_hwinfo *hw = rvu->hw; 261 struct nix_txsch *txsch; 262 struct nix_hw *nix_hw; 263 u16 map_func; 264 265 nix_hw = get_nix_hw(rvu->hw, blkaddr); 266 if (!nix_hw) 267 return false; 268 269 txsch = &nix_hw->txsch[lvl]; 270 /* Check out of bounds */ 271 if (schq >= txsch->schq.max) 272 return false; 273 274 mutex_lock(&rvu->rsrc_lock); 275 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 276 mutex_unlock(&rvu->rsrc_lock); 277 278 /* TLs aggegating traffic are shared across PF and VFs */ 279 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 280 if (rvu_get_pf(map_func) != rvu_get_pf(pcifunc)) 281 return false; 282 else 283 return true; 284 } 285 286 if (map_func != pcifunc) 287 return false; 288 289 return true; 290 } 291 292 static int nix_interface_init(struct rvu *rvu, u16 pcifunc, int type, int nixlf, 293 struct nix_lf_alloc_rsp *rsp, bool loop) 294 { 295 struct rvu_pfvf *parent_pf, *pfvf = rvu_get_pfvf(rvu, pcifunc); 296 u16 req_chan_base, req_chan_end, req_chan_cnt; 297 struct rvu_hwinfo *hw = rvu->hw; 298 struct sdp_node_info *sdp_info; 299 int pkind, pf, vf, lbkid, vfid; 300 u8 cgx_id, lmac_id; 301 bool from_vf; 302 int err; 303 304 pf = rvu_get_pf(pcifunc); 305 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 306 type != NIX_INTF_TYPE_SDP) 307 return 0; 308 309 switch (type) { 310 case NIX_INTF_TYPE_CGX: 311 pfvf->cgx_lmac = rvu->pf2cgxlmac_map[pf]; 312 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 313 314 pkind = rvu_npc_get_pkind(rvu, pf); 315 if (pkind < 0) { 316 dev_err(rvu->dev, 317 "PF_Func 0x%x: Invalid pkind\n", pcifunc); 318 return -EINVAL; 319 } 320 pfvf->rx_chan_base = rvu_nix_chan_cgx(rvu, cgx_id, lmac_id, 0); 321 pfvf->tx_chan_base = pfvf->rx_chan_base; 322 pfvf->rx_chan_cnt = 1; 323 pfvf->tx_chan_cnt = 1; 324 rsp->tx_link = cgx_id * hw->lmac_per_cgx + lmac_id; 325 326 cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id, pkind); 327 rvu_npc_set_pkind(rvu, pkind, pfvf); 328 329 break; 330 case NIX_INTF_TYPE_LBK: 331 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 332 333 /* If NIX1 block is present on the silicon then NIXes are 334 * assigned alternatively for lbk interfaces. NIX0 should 335 * send packets on lbk link 1 channels and NIX1 should send 336 * on lbk link 0 channels for the communication between 337 * NIX0 and NIX1. 338 */ 339 lbkid = 0; 340 if (rvu->hw->lbk_links > 1) 341 lbkid = vf & 0x1 ? 0 : 1; 342 343 /* By default NIX0 is configured to send packet on lbk link 1 344 * (which corresponds to LBK1), same packet will receive on 345 * NIX1 over lbk link 0. If NIX1 sends packet on lbk link 0 346 * (which corresponds to LBK2) packet will receive on NIX0 lbk 347 * link 1. 348 * But if lbk links for NIX0 and NIX1 are negated, i.e NIX0 349 * transmits and receives on lbk link 0, whick corresponds 350 * to LBK1 block, back to back connectivity between NIX and 351 * LBK can be achieved (which is similar to 96xx) 352 * 353 * RX TX 354 * NIX0 lbk link 1 (LBK2) 1 (LBK1) 355 * NIX0 lbk link 0 (LBK0) 0 (LBK0) 356 * NIX1 lbk link 0 (LBK1) 0 (LBK2) 357 * NIX1 lbk link 1 (LBK3) 1 (LBK3) 358 */ 359 if (loop) 360 lbkid = !lbkid; 361 362 /* Note that AF's VFs work in pairs and talk over consecutive 363 * loopback channels.Therefore if odd number of AF VFs are 364 * enabled then the last VF remains with no pair. 365 */ 366 pfvf->rx_chan_base = rvu_nix_chan_lbk(rvu, lbkid, vf); 367 pfvf->tx_chan_base = vf & 0x1 ? 368 rvu_nix_chan_lbk(rvu, lbkid, vf - 1) : 369 rvu_nix_chan_lbk(rvu, lbkid, vf + 1); 370 pfvf->rx_chan_cnt = 1; 371 pfvf->tx_chan_cnt = 1; 372 rsp->tx_link = hw->cgx_links + lbkid; 373 pfvf->lbkid = lbkid; 374 rvu_npc_set_pkind(rvu, NPC_RX_LBK_PKIND, pfvf); 375 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 376 pfvf->rx_chan_base, 377 pfvf->rx_chan_cnt); 378 379 break; 380 case NIX_INTF_TYPE_SDP: 381 from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 382 parent_pf = &rvu->pf[rvu_get_pf(pcifunc)]; 383 sdp_info = parent_pf->sdp_info; 384 if (!sdp_info) { 385 dev_err(rvu->dev, "Invalid sdp_info pointer\n"); 386 return -EINVAL; 387 } 388 if (from_vf) { 389 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn + 390 sdp_info->num_pf_rings; 391 vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1; 392 for (vfid = 0; vfid < vf; vfid++) 393 req_chan_base += sdp_info->vf_rings[vfid]; 394 req_chan_cnt = sdp_info->vf_rings[vf]; 395 req_chan_end = req_chan_base + req_chan_cnt - 1; 396 if (req_chan_base < rvu_nix_chan_sdp(rvu, 0) || 397 req_chan_end > rvu_nix_chan_sdp(rvu, 255)) { 398 dev_err(rvu->dev, 399 "PF_Func 0x%x: Invalid channel base and count\n", 400 pcifunc); 401 return -EINVAL; 402 } 403 } else { 404 req_chan_base = rvu_nix_chan_sdp(rvu, 0) + sdp_info->pf_srn; 405 req_chan_cnt = sdp_info->num_pf_rings; 406 } 407 408 pfvf->rx_chan_base = req_chan_base; 409 pfvf->rx_chan_cnt = req_chan_cnt; 410 pfvf->tx_chan_base = pfvf->rx_chan_base; 411 pfvf->tx_chan_cnt = pfvf->rx_chan_cnt; 412 413 rsp->tx_link = hw->cgx_links + hw->lbk_links; 414 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 415 pfvf->rx_chan_base, 416 pfvf->rx_chan_cnt); 417 break; 418 } 419 420 /* Add a UCAST forwarding rule in MCAM with this NIXLF attached 421 * RVU PF/VF's MAC address. 422 */ 423 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 424 pfvf->rx_chan_base, pfvf->mac_addr); 425 426 /* Add this PF_FUNC to bcast pkt replication list */ 427 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, true); 428 if (err) { 429 dev_err(rvu->dev, 430 "Bcast list, failed to enable PF_FUNC 0x%x\n", 431 pcifunc); 432 return err; 433 } 434 /* Install MCAM rule matching Ethernet broadcast mac address */ 435 rvu_npc_install_bcast_match_entry(rvu, pcifunc, 436 nixlf, pfvf->rx_chan_base); 437 438 pfvf->maxlen = NIC_HW_MIN_FRS; 439 pfvf->minlen = NIC_HW_MIN_FRS; 440 441 return 0; 442 } 443 444 static void nix_interface_deinit(struct rvu *rvu, u16 pcifunc, u8 nixlf) 445 { 446 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 447 int err; 448 449 pfvf->maxlen = 0; 450 pfvf->minlen = 0; 451 452 /* Remove this PF_FUNC from bcast pkt replication list */ 453 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_BCAST_ENTRY, false); 454 if (err) { 455 dev_err(rvu->dev, 456 "Bcast list, failed to disable PF_FUNC 0x%x\n", 457 pcifunc); 458 } 459 460 /* Free and disable any MCAM entries used by this NIX LF */ 461 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 462 463 /* Disable DMAC filters used */ 464 rvu_cgx_disable_dmac_entries(rvu, pcifunc); 465 } 466 467 int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu, 468 struct nix_bp_cfg_req *req, 469 struct msg_rsp *rsp) 470 { 471 u16 pcifunc = req->hdr.pcifunc; 472 struct rvu_pfvf *pfvf; 473 int blkaddr, pf, type; 474 u16 chan_base, chan; 475 u64 cfg; 476 477 pf = rvu_get_pf(pcifunc); 478 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 479 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK) 480 return 0; 481 482 pfvf = rvu_get_pfvf(rvu, pcifunc); 483 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 484 485 chan_base = pfvf->rx_chan_base + req->chan_base; 486 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 487 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan)); 488 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan), 489 cfg & ~BIT_ULL(16)); 490 } 491 return 0; 492 } 493 494 static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req, 495 int type, int chan_id) 496 { 497 int bpid, blkaddr, lmac_chan_cnt, sdp_chan_cnt; 498 u16 cgx_bpid_cnt, lbk_bpid_cnt, sdp_bpid_cnt; 499 struct rvu_hwinfo *hw = rvu->hw; 500 struct rvu_pfvf *pfvf; 501 u8 cgx_id, lmac_id; 502 u64 cfg; 503 504 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc); 505 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); 506 lmac_chan_cnt = cfg & 0xFF; 507 508 cgx_bpid_cnt = hw->cgx_links * lmac_chan_cnt; 509 lbk_bpid_cnt = hw->lbk_links * ((cfg >> 16) & 0xFF); 510 511 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 512 sdp_chan_cnt = cfg & 0xFFF; 513 sdp_bpid_cnt = hw->sdp_links * sdp_chan_cnt; 514 515 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 516 517 /* Backpressure IDs range division 518 * CGX channles are mapped to (0 - 191) BPIDs 519 * LBK channles are mapped to (192 - 255) BPIDs 520 * SDP channles are mapped to (256 - 511) BPIDs 521 * 522 * Lmac channles and bpids mapped as follows 523 * cgx(0)_lmac(0)_chan(0 - 15) = bpid(0 - 15) 524 * cgx(0)_lmac(1)_chan(0 - 15) = bpid(16 - 31) .... 525 * cgx(1)_lmac(0)_chan(0 - 15) = bpid(64 - 79) .... 526 */ 527 switch (type) { 528 case NIX_INTF_TYPE_CGX: 529 if ((req->chan_base + req->chan_cnt) > 16) 530 return -EINVAL; 531 rvu_get_cgx_lmac_id(pfvf->cgx_lmac, &cgx_id, &lmac_id); 532 /* Assign bpid based on cgx, lmac and chan id */ 533 bpid = (cgx_id * hw->lmac_per_cgx * lmac_chan_cnt) + 534 (lmac_id * lmac_chan_cnt) + req->chan_base; 535 536 if (req->bpid_per_chan) 537 bpid += chan_id; 538 if (bpid > cgx_bpid_cnt) 539 return -EINVAL; 540 break; 541 542 case NIX_INTF_TYPE_LBK: 543 if ((req->chan_base + req->chan_cnt) > 63) 544 return -EINVAL; 545 bpid = cgx_bpid_cnt + req->chan_base; 546 if (req->bpid_per_chan) 547 bpid += chan_id; 548 if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt)) 549 return -EINVAL; 550 break; 551 case NIX_INTF_TYPE_SDP: 552 if ((req->chan_base + req->chan_cnt) > 255) 553 return -EINVAL; 554 555 bpid = sdp_bpid_cnt + req->chan_base; 556 if (req->bpid_per_chan) 557 bpid += chan_id; 558 559 if (bpid > (cgx_bpid_cnt + lbk_bpid_cnt + sdp_bpid_cnt)) 560 return -EINVAL; 561 break; 562 default: 563 return -EINVAL; 564 } 565 return bpid; 566 } 567 568 int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu, 569 struct nix_bp_cfg_req *req, 570 struct nix_bp_cfg_rsp *rsp) 571 { 572 int blkaddr, pf, type, chan_id = 0; 573 u16 pcifunc = req->hdr.pcifunc; 574 struct rvu_pfvf *pfvf; 575 u16 chan_base, chan; 576 s16 bpid, bpid_base; 577 u64 cfg; 578 579 pf = rvu_get_pf(pcifunc); 580 type = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 581 if (is_sdp_pfvf(pcifunc)) 582 type = NIX_INTF_TYPE_SDP; 583 584 /* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */ 585 if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK && 586 type != NIX_INTF_TYPE_SDP) 587 return 0; 588 589 pfvf = rvu_get_pfvf(rvu, pcifunc); 590 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 591 592 bpid_base = rvu_nix_get_bpid(rvu, req, type, chan_id); 593 chan_base = pfvf->rx_chan_base + req->chan_base; 594 bpid = bpid_base; 595 596 for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) { 597 if (bpid < 0) { 598 dev_warn(rvu->dev, "Fail to enable backpressure\n"); 599 return -EINVAL; 600 } 601 602 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan)); 603 cfg &= ~GENMASK_ULL(8, 0); 604 rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan), 605 cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16)); 606 chan_id++; 607 bpid = rvu_nix_get_bpid(rvu, req, type, chan_id); 608 } 609 610 for (chan = 0; chan < req->chan_cnt; chan++) { 611 /* Map channel and bpid assign to it */ 612 rsp->chan_bpid[chan] = ((req->chan_base + chan) & 0x7F) << 10 | 613 (bpid_base & 0x3FF); 614 if (req->bpid_per_chan) 615 bpid_base++; 616 } 617 rsp->chan_cnt = req->chan_cnt; 618 619 return 0; 620 } 621 622 static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr, 623 u64 format, bool v4, u64 *fidx) 624 { 625 struct nix_lso_format field = {0}; 626 627 /* IP's Length field */ 628 field.layer = NIX_TXLAYER_OL3; 629 /* In ipv4, length field is at offset 2 bytes, for ipv6 it's 4 */ 630 field.offset = v4 ? 2 : 4; 631 field.sizem1 = 1; /* i.e 2 bytes */ 632 field.alg = NIX_LSOALG_ADD_PAYLEN; 633 rvu_write64(rvu, blkaddr, 634 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 635 *(u64 *)&field); 636 637 /* No ID field in IPv6 header */ 638 if (!v4) 639 return; 640 641 /* IP's ID field */ 642 field.layer = NIX_TXLAYER_OL3; 643 field.offset = 4; 644 field.sizem1 = 1; /* i.e 2 bytes */ 645 field.alg = NIX_LSOALG_ADD_SEGNUM; 646 rvu_write64(rvu, blkaddr, 647 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 648 *(u64 *)&field); 649 } 650 651 static void nix_setup_lso_tso_l4(struct rvu *rvu, int blkaddr, 652 u64 format, u64 *fidx) 653 { 654 struct nix_lso_format field = {0}; 655 656 /* TCP's sequence number field */ 657 field.layer = NIX_TXLAYER_OL4; 658 field.offset = 4; 659 field.sizem1 = 3; /* i.e 4 bytes */ 660 field.alg = NIX_LSOALG_ADD_OFFSET; 661 rvu_write64(rvu, blkaddr, 662 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 663 *(u64 *)&field); 664 665 /* TCP's flags field */ 666 field.layer = NIX_TXLAYER_OL4; 667 field.offset = 12; 668 field.sizem1 = 1; /* 2 bytes */ 669 field.alg = NIX_LSOALG_TCP_FLAGS; 670 rvu_write64(rvu, blkaddr, 671 NIX_AF_LSO_FORMATX_FIELDX(format, (*fidx)++), 672 *(u64 *)&field); 673 } 674 675 static void nix_setup_lso(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 676 { 677 u64 cfg, idx, fidx = 0; 678 679 /* Get max HW supported format indices */ 680 cfg = (rvu_read64(rvu, blkaddr, NIX_AF_CONST1) >> 48) & 0xFF; 681 nix_hw->lso.total = cfg; 682 683 /* Enable LSO */ 684 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LSO_CFG); 685 /* For TSO, set first and middle segment flags to 686 * mask out PSH, RST & FIN flags in TCP packet 687 */ 688 cfg &= ~((0xFFFFULL << 32) | (0xFFFFULL << 16)); 689 cfg |= (0xFFF2ULL << 32) | (0xFFF2ULL << 16); 690 rvu_write64(rvu, blkaddr, NIX_AF_LSO_CFG, cfg | BIT_ULL(63)); 691 692 /* Setup default static LSO formats 693 * 694 * Configure format fields for TCPv4 segmentation offload 695 */ 696 idx = NIX_LSO_FORMAT_IDX_TSOV4; 697 nix_setup_lso_tso_l3(rvu, blkaddr, idx, true, &fidx); 698 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 699 700 /* Set rest of the fields to NOP */ 701 for (; fidx < 8; fidx++) { 702 rvu_write64(rvu, blkaddr, 703 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 704 } 705 nix_hw->lso.in_use++; 706 707 /* Configure format fields for TCPv6 segmentation offload */ 708 idx = NIX_LSO_FORMAT_IDX_TSOV6; 709 fidx = 0; 710 nix_setup_lso_tso_l3(rvu, blkaddr, idx, false, &fidx); 711 nix_setup_lso_tso_l4(rvu, blkaddr, idx, &fidx); 712 713 /* Set rest of the fields to NOP */ 714 for (; fidx < 8; fidx++) { 715 rvu_write64(rvu, blkaddr, 716 NIX_AF_LSO_FORMATX_FIELDX(idx, fidx), 0x0ULL); 717 } 718 nix_hw->lso.in_use++; 719 } 720 721 static void nix_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) 722 { 723 kfree(pfvf->rq_bmap); 724 kfree(pfvf->sq_bmap); 725 kfree(pfvf->cq_bmap); 726 if (pfvf->rq_ctx) 727 qmem_free(rvu->dev, pfvf->rq_ctx); 728 if (pfvf->sq_ctx) 729 qmem_free(rvu->dev, pfvf->sq_ctx); 730 if (pfvf->cq_ctx) 731 qmem_free(rvu->dev, pfvf->cq_ctx); 732 if (pfvf->rss_ctx) 733 qmem_free(rvu->dev, pfvf->rss_ctx); 734 if (pfvf->nix_qints_ctx) 735 qmem_free(rvu->dev, pfvf->nix_qints_ctx); 736 if (pfvf->cq_ints_ctx) 737 qmem_free(rvu->dev, pfvf->cq_ints_ctx); 738 739 pfvf->rq_bmap = NULL; 740 pfvf->cq_bmap = NULL; 741 pfvf->sq_bmap = NULL; 742 pfvf->rq_ctx = NULL; 743 pfvf->sq_ctx = NULL; 744 pfvf->cq_ctx = NULL; 745 pfvf->rss_ctx = NULL; 746 pfvf->nix_qints_ctx = NULL; 747 pfvf->cq_ints_ctx = NULL; 748 } 749 750 static int nixlf_rss_ctx_init(struct rvu *rvu, int blkaddr, 751 struct rvu_pfvf *pfvf, int nixlf, 752 int rss_sz, int rss_grps, int hwctx_size, 753 u64 way_mask, bool tag_lsb_as_adder) 754 { 755 int err, grp, num_indices; 756 u64 val; 757 758 /* RSS is not requested for this NIXLF */ 759 if (!rss_sz) 760 return 0; 761 num_indices = rss_sz * rss_grps; 762 763 /* Alloc NIX RSS HW context memory and config the base */ 764 err = qmem_alloc(rvu->dev, &pfvf->rss_ctx, num_indices, hwctx_size); 765 if (err) 766 return err; 767 768 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_BASE(nixlf), 769 (u64)pfvf->rss_ctx->iova); 770 771 /* Config full RSS table size, enable RSS and caching */ 772 val = BIT_ULL(36) | BIT_ULL(4) | way_mask << 20 | 773 ilog2(num_indices / MAX_RSS_INDIR_TBL_SIZE); 774 775 if (tag_lsb_as_adder) 776 val |= BIT_ULL(5); 777 778 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf), val); 779 /* Config RSS group offset and sizes */ 780 for (grp = 0; grp < rss_grps; grp++) 781 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RSS_GRPX(nixlf, grp), 782 ((ilog2(rss_sz) - 1) << 16) | (rss_sz * grp)); 783 return 0; 784 } 785 786 static int nix_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, 787 struct nix_aq_inst_s *inst) 788 { 789 struct admin_queue *aq = block->aq; 790 struct nix_aq_res_s *result; 791 int timeout = 1000; 792 u64 reg, head; 793 int ret; 794 795 result = (struct nix_aq_res_s *)aq->res->base; 796 797 /* Get current head pointer where to append this instruction */ 798 reg = rvu_read64(rvu, block->addr, NIX_AF_AQ_STATUS); 799 head = (reg >> 4) & AQ_PTR_MASK; 800 801 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), 802 (void *)inst, aq->inst->entry_sz); 803 memset(result, 0, sizeof(*result)); 804 /* sync into memory */ 805 wmb(); 806 807 /* Ring the doorbell and wait for result */ 808 rvu_write64(rvu, block->addr, NIX_AF_AQ_DOOR, 1); 809 while (result->compcode == NIX_AQ_COMP_NOTDONE) { 810 cpu_relax(); 811 udelay(1); 812 timeout--; 813 if (!timeout) 814 return -EBUSY; 815 } 816 817 if (result->compcode != NIX_AQ_COMP_GOOD) { 818 /* TODO: Replace this with some error code */ 819 if (result->compcode == NIX_AQ_COMP_CTX_FAULT || 820 result->compcode == NIX_AQ_COMP_LOCKERR || 821 result->compcode == NIX_AQ_COMP_CTX_POISON) { 822 ret = rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_RX); 823 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX0_TX); 824 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_RX); 825 ret |= rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NIX1_TX); 826 if (ret) 827 dev_err(rvu->dev, 828 "%s: Not able to unlock cachelines\n", __func__); 829 } 830 831 return -EBUSY; 832 } 833 834 return 0; 835 } 836 837 static int rvu_nix_blk_aq_enq_inst(struct rvu *rvu, struct nix_hw *nix_hw, 838 struct nix_aq_enq_req *req, 839 struct nix_aq_enq_rsp *rsp) 840 { 841 struct rvu_hwinfo *hw = rvu->hw; 842 u16 pcifunc = req->hdr.pcifunc; 843 int nixlf, blkaddr, rc = 0; 844 struct nix_aq_inst_s inst; 845 struct rvu_block *block; 846 struct admin_queue *aq; 847 struct rvu_pfvf *pfvf; 848 void *ctx, *mask; 849 bool ena; 850 u64 cfg; 851 852 blkaddr = nix_hw->blkaddr; 853 block = &hw->block[blkaddr]; 854 aq = block->aq; 855 if (!aq) { 856 dev_warn(rvu->dev, "%s: NIX AQ not initialized\n", __func__); 857 return NIX_AF_ERR_AQ_ENQUEUE; 858 } 859 860 pfvf = rvu_get_pfvf(rvu, pcifunc); 861 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 862 863 /* Skip NIXLF check for broadcast MCE entry and bandwidth profile 864 * operations done by AF itself. 865 */ 866 if (!((!rsp && req->ctype == NIX_AQ_CTYPE_MCE) || 867 (req->ctype == NIX_AQ_CTYPE_BANDPROF && !pcifunc))) { 868 if (!pfvf->nixlf || nixlf < 0) 869 return NIX_AF_ERR_AF_LF_INVALID; 870 } 871 872 switch (req->ctype) { 873 case NIX_AQ_CTYPE_RQ: 874 /* Check if index exceeds max no of queues */ 875 if (!pfvf->rq_ctx || req->qidx >= pfvf->rq_ctx->qsize) 876 rc = NIX_AF_ERR_AQ_ENQUEUE; 877 break; 878 case NIX_AQ_CTYPE_SQ: 879 if (!pfvf->sq_ctx || req->qidx >= pfvf->sq_ctx->qsize) 880 rc = NIX_AF_ERR_AQ_ENQUEUE; 881 break; 882 case NIX_AQ_CTYPE_CQ: 883 if (!pfvf->cq_ctx || req->qidx >= pfvf->cq_ctx->qsize) 884 rc = NIX_AF_ERR_AQ_ENQUEUE; 885 break; 886 case NIX_AQ_CTYPE_RSS: 887 /* Check if RSS is enabled and qidx is within range */ 888 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RSS_CFG(nixlf)); 889 if (!(cfg & BIT_ULL(4)) || !pfvf->rss_ctx || 890 (req->qidx >= (256UL << (cfg & 0xF)))) 891 rc = NIX_AF_ERR_AQ_ENQUEUE; 892 break; 893 case NIX_AQ_CTYPE_MCE: 894 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG); 895 896 /* Check if index exceeds MCE list length */ 897 if (!nix_hw->mcast.mce_ctx || 898 (req->qidx >= (256UL << (cfg & 0xF)))) 899 rc = NIX_AF_ERR_AQ_ENQUEUE; 900 901 /* Adding multicast lists for requests from PF/VFs is not 902 * yet supported, so ignore this. 903 */ 904 if (rsp) 905 rc = NIX_AF_ERR_AQ_ENQUEUE; 906 break; 907 case NIX_AQ_CTYPE_BANDPROF: 908 if (nix_verify_bandprof((struct nix_cn10k_aq_enq_req *)req, 909 nix_hw, pcifunc)) 910 rc = NIX_AF_ERR_INVALID_BANDPROF; 911 break; 912 default: 913 rc = NIX_AF_ERR_AQ_ENQUEUE; 914 } 915 916 if (rc) 917 return rc; 918 919 /* Check if SQ pointed SMQ belongs to this PF/VF or not */ 920 if (req->ctype == NIX_AQ_CTYPE_SQ && 921 ((req->op == NIX_AQ_INSTOP_INIT && req->sq.ena) || 922 (req->op == NIX_AQ_INSTOP_WRITE && 923 req->sq_mask.ena && req->sq_mask.smq && req->sq.ena))) { 924 if (!is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_SMQ, 925 pcifunc, req->sq.smq)) 926 return NIX_AF_ERR_AQ_ENQUEUE; 927 } 928 929 memset(&inst, 0, sizeof(struct nix_aq_inst_s)); 930 inst.lf = nixlf; 931 inst.cindex = req->qidx; 932 inst.ctype = req->ctype; 933 inst.op = req->op; 934 /* Currently we are not supporting enqueuing multiple instructions, 935 * so always choose first entry in result memory. 936 */ 937 inst.res_addr = (u64)aq->res->iova; 938 939 /* Hardware uses same aq->res->base for updating result of 940 * previous instruction hence wait here till it is done. 941 */ 942 spin_lock(&aq->lock); 943 944 /* Clean result + context memory */ 945 memset(aq->res->base, 0, aq->res->entry_sz); 946 /* Context needs to be written at RES_ADDR + 128 */ 947 ctx = aq->res->base + 128; 948 /* Mask needs to be written at RES_ADDR + 256 */ 949 mask = aq->res->base + 256; 950 951 switch (req->op) { 952 case NIX_AQ_INSTOP_WRITE: 953 if (req->ctype == NIX_AQ_CTYPE_RQ) 954 memcpy(mask, &req->rq_mask, 955 sizeof(struct nix_rq_ctx_s)); 956 else if (req->ctype == NIX_AQ_CTYPE_SQ) 957 memcpy(mask, &req->sq_mask, 958 sizeof(struct nix_sq_ctx_s)); 959 else if (req->ctype == NIX_AQ_CTYPE_CQ) 960 memcpy(mask, &req->cq_mask, 961 sizeof(struct nix_cq_ctx_s)); 962 else if (req->ctype == NIX_AQ_CTYPE_RSS) 963 memcpy(mask, &req->rss_mask, 964 sizeof(struct nix_rsse_s)); 965 else if (req->ctype == NIX_AQ_CTYPE_MCE) 966 memcpy(mask, &req->mce_mask, 967 sizeof(struct nix_rx_mce_s)); 968 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 969 memcpy(mask, &req->prof_mask, 970 sizeof(struct nix_bandprof_s)); 971 fallthrough; 972 case NIX_AQ_INSTOP_INIT: 973 if (req->ctype == NIX_AQ_CTYPE_RQ) 974 memcpy(ctx, &req->rq, sizeof(struct nix_rq_ctx_s)); 975 else if (req->ctype == NIX_AQ_CTYPE_SQ) 976 memcpy(ctx, &req->sq, sizeof(struct nix_sq_ctx_s)); 977 else if (req->ctype == NIX_AQ_CTYPE_CQ) 978 memcpy(ctx, &req->cq, sizeof(struct nix_cq_ctx_s)); 979 else if (req->ctype == NIX_AQ_CTYPE_RSS) 980 memcpy(ctx, &req->rss, sizeof(struct nix_rsse_s)); 981 else if (req->ctype == NIX_AQ_CTYPE_MCE) 982 memcpy(ctx, &req->mce, sizeof(struct nix_rx_mce_s)); 983 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 984 memcpy(ctx, &req->prof, sizeof(struct nix_bandprof_s)); 985 break; 986 case NIX_AQ_INSTOP_NOP: 987 case NIX_AQ_INSTOP_READ: 988 case NIX_AQ_INSTOP_LOCK: 989 case NIX_AQ_INSTOP_UNLOCK: 990 break; 991 default: 992 rc = NIX_AF_ERR_AQ_ENQUEUE; 993 spin_unlock(&aq->lock); 994 return rc; 995 } 996 997 /* Submit the instruction to AQ */ 998 rc = nix_aq_enqueue_wait(rvu, block, &inst); 999 if (rc) { 1000 spin_unlock(&aq->lock); 1001 return rc; 1002 } 1003 1004 /* Set RQ/SQ/CQ bitmap if respective queue hw context is enabled */ 1005 if (req->op == NIX_AQ_INSTOP_INIT) { 1006 if (req->ctype == NIX_AQ_CTYPE_RQ && req->rq.ena) 1007 __set_bit(req->qidx, pfvf->rq_bmap); 1008 if (req->ctype == NIX_AQ_CTYPE_SQ && req->sq.ena) 1009 __set_bit(req->qidx, pfvf->sq_bmap); 1010 if (req->ctype == NIX_AQ_CTYPE_CQ && req->cq.ena) 1011 __set_bit(req->qidx, pfvf->cq_bmap); 1012 } 1013 1014 if (req->op == NIX_AQ_INSTOP_WRITE) { 1015 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1016 ena = (req->rq.ena & req->rq_mask.ena) | 1017 (test_bit(req->qidx, pfvf->rq_bmap) & 1018 ~req->rq_mask.ena); 1019 if (ena) 1020 __set_bit(req->qidx, pfvf->rq_bmap); 1021 else 1022 __clear_bit(req->qidx, pfvf->rq_bmap); 1023 } 1024 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1025 ena = (req->rq.ena & req->sq_mask.ena) | 1026 (test_bit(req->qidx, pfvf->sq_bmap) & 1027 ~req->sq_mask.ena); 1028 if (ena) 1029 __set_bit(req->qidx, pfvf->sq_bmap); 1030 else 1031 __clear_bit(req->qidx, pfvf->sq_bmap); 1032 } 1033 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1034 ena = (req->rq.ena & req->cq_mask.ena) | 1035 (test_bit(req->qidx, pfvf->cq_bmap) & 1036 ~req->cq_mask.ena); 1037 if (ena) 1038 __set_bit(req->qidx, pfvf->cq_bmap); 1039 else 1040 __clear_bit(req->qidx, pfvf->cq_bmap); 1041 } 1042 } 1043 1044 if (rsp) { 1045 /* Copy read context into mailbox */ 1046 if (req->op == NIX_AQ_INSTOP_READ) { 1047 if (req->ctype == NIX_AQ_CTYPE_RQ) 1048 memcpy(&rsp->rq, ctx, 1049 sizeof(struct nix_rq_ctx_s)); 1050 else if (req->ctype == NIX_AQ_CTYPE_SQ) 1051 memcpy(&rsp->sq, ctx, 1052 sizeof(struct nix_sq_ctx_s)); 1053 else if (req->ctype == NIX_AQ_CTYPE_CQ) 1054 memcpy(&rsp->cq, ctx, 1055 sizeof(struct nix_cq_ctx_s)); 1056 else if (req->ctype == NIX_AQ_CTYPE_RSS) 1057 memcpy(&rsp->rss, ctx, 1058 sizeof(struct nix_rsse_s)); 1059 else if (req->ctype == NIX_AQ_CTYPE_MCE) 1060 memcpy(&rsp->mce, ctx, 1061 sizeof(struct nix_rx_mce_s)); 1062 else if (req->ctype == NIX_AQ_CTYPE_BANDPROF) 1063 memcpy(&rsp->prof, ctx, 1064 sizeof(struct nix_bandprof_s)); 1065 } 1066 } 1067 1068 spin_unlock(&aq->lock); 1069 return 0; 1070 } 1071 1072 static int rvu_nix_verify_aq_ctx(struct rvu *rvu, struct nix_hw *nix_hw, 1073 struct nix_aq_enq_req *req, u8 ctype) 1074 { 1075 struct nix_cn10k_aq_enq_req aq_req; 1076 struct nix_cn10k_aq_enq_rsp aq_rsp; 1077 int rc, word; 1078 1079 if (req->ctype != NIX_AQ_CTYPE_CQ) 1080 return 0; 1081 1082 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 1083 req->hdr.pcifunc, ctype, req->qidx); 1084 if (rc) { 1085 dev_err(rvu->dev, 1086 "%s: Failed to fetch %s%d context of PFFUNC 0x%x\n", 1087 __func__, nix_get_ctx_name(ctype), req->qidx, 1088 req->hdr.pcifunc); 1089 return rc; 1090 } 1091 1092 /* Make copy of original context & mask which are required 1093 * for resubmission 1094 */ 1095 memcpy(&aq_req.cq_mask, &req->cq_mask, sizeof(struct nix_cq_ctx_s)); 1096 memcpy(&aq_req.cq, &req->cq, sizeof(struct nix_cq_ctx_s)); 1097 1098 /* exclude fields which HW can update */ 1099 aq_req.cq_mask.cq_err = 0; 1100 aq_req.cq_mask.wrptr = 0; 1101 aq_req.cq_mask.tail = 0; 1102 aq_req.cq_mask.head = 0; 1103 aq_req.cq_mask.avg_level = 0; 1104 aq_req.cq_mask.update_time = 0; 1105 aq_req.cq_mask.substream = 0; 1106 1107 /* Context mask (cq_mask) holds mask value of fields which 1108 * are changed in AQ WRITE operation. 1109 * for example cq.drop = 0xa; 1110 * cq_mask.drop = 0xff; 1111 * Below logic performs '&' between cq and cq_mask so that non 1112 * updated fields are masked out for request and response 1113 * comparison 1114 */ 1115 for (word = 0; word < sizeof(struct nix_cq_ctx_s) / sizeof(u64); 1116 word++) { 1117 *(u64 *)((u8 *)&aq_rsp.cq + word * 8) &= 1118 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1119 *(u64 *)((u8 *)&aq_req.cq + word * 8) &= 1120 (*(u64 *)((u8 *)&aq_req.cq_mask + word * 8)); 1121 } 1122 1123 if (memcmp(&aq_req.cq, &aq_rsp.cq, sizeof(struct nix_cq_ctx_s))) 1124 return NIX_AF_ERR_AQ_CTX_RETRY_WRITE; 1125 1126 return 0; 1127 } 1128 1129 static int rvu_nix_aq_enq_inst(struct rvu *rvu, struct nix_aq_enq_req *req, 1130 struct nix_aq_enq_rsp *rsp) 1131 { 1132 struct nix_hw *nix_hw; 1133 int err, retries = 5; 1134 int blkaddr; 1135 1136 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, req->hdr.pcifunc); 1137 if (blkaddr < 0) 1138 return NIX_AF_ERR_AF_LF_INVALID; 1139 1140 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1141 if (!nix_hw) 1142 return NIX_AF_ERR_INVALID_NIXBLK; 1143 1144 retry: 1145 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, req, rsp); 1146 1147 /* HW errata 'AQ Modification to CQ could be discarded on heavy traffic' 1148 * As a work around perfrom CQ context read after each AQ write. If AQ 1149 * read shows AQ write is not updated perform AQ write again. 1150 */ 1151 if (!err && req->op == NIX_AQ_INSTOP_WRITE) { 1152 err = rvu_nix_verify_aq_ctx(rvu, nix_hw, req, NIX_AQ_CTYPE_CQ); 1153 if (err == NIX_AF_ERR_AQ_CTX_RETRY_WRITE) { 1154 if (retries--) 1155 goto retry; 1156 else 1157 return NIX_AF_ERR_CQ_CTX_WRITE_ERR; 1158 } 1159 } 1160 1161 return err; 1162 } 1163 1164 static const char *nix_get_ctx_name(int ctype) 1165 { 1166 switch (ctype) { 1167 case NIX_AQ_CTYPE_CQ: 1168 return "CQ"; 1169 case NIX_AQ_CTYPE_SQ: 1170 return "SQ"; 1171 case NIX_AQ_CTYPE_RQ: 1172 return "RQ"; 1173 case NIX_AQ_CTYPE_RSS: 1174 return "RSS"; 1175 } 1176 return ""; 1177 } 1178 1179 static int nix_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) 1180 { 1181 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 1182 struct nix_aq_enq_req aq_req; 1183 unsigned long *bmap; 1184 int qidx, q_cnt = 0; 1185 int err = 0, rc; 1186 1187 if (!pfvf->cq_ctx || !pfvf->sq_ctx || !pfvf->rq_ctx) 1188 return NIX_AF_ERR_AQ_ENQUEUE; 1189 1190 memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); 1191 aq_req.hdr.pcifunc = req->hdr.pcifunc; 1192 1193 if (req->ctype == NIX_AQ_CTYPE_CQ) { 1194 aq_req.cq.ena = 0; 1195 aq_req.cq_mask.ena = 1; 1196 aq_req.cq.bp_ena = 0; 1197 aq_req.cq_mask.bp_ena = 1; 1198 q_cnt = pfvf->cq_ctx->qsize; 1199 bmap = pfvf->cq_bmap; 1200 } 1201 if (req->ctype == NIX_AQ_CTYPE_SQ) { 1202 aq_req.sq.ena = 0; 1203 aq_req.sq_mask.ena = 1; 1204 q_cnt = pfvf->sq_ctx->qsize; 1205 bmap = pfvf->sq_bmap; 1206 } 1207 if (req->ctype == NIX_AQ_CTYPE_RQ) { 1208 aq_req.rq.ena = 0; 1209 aq_req.rq_mask.ena = 1; 1210 q_cnt = pfvf->rq_ctx->qsize; 1211 bmap = pfvf->rq_bmap; 1212 } 1213 1214 aq_req.ctype = req->ctype; 1215 aq_req.op = NIX_AQ_INSTOP_WRITE; 1216 1217 for (qidx = 0; qidx < q_cnt; qidx++) { 1218 if (!test_bit(qidx, bmap)) 1219 continue; 1220 aq_req.qidx = qidx; 1221 rc = rvu_nix_aq_enq_inst(rvu, &aq_req, NULL); 1222 if (rc) { 1223 err = rc; 1224 dev_err(rvu->dev, "Failed to disable %s:%d context\n", 1225 nix_get_ctx_name(req->ctype), qidx); 1226 } 1227 } 1228 1229 return err; 1230 } 1231 1232 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 1233 static int nix_lf_hwctx_lockdown(struct rvu *rvu, struct nix_aq_enq_req *req) 1234 { 1235 struct nix_aq_enq_req lock_ctx_req; 1236 int err; 1237 1238 if (req->op != NIX_AQ_INSTOP_INIT) 1239 return 0; 1240 1241 if (req->ctype == NIX_AQ_CTYPE_MCE || 1242 req->ctype == NIX_AQ_CTYPE_DYNO) 1243 return 0; 1244 1245 memset(&lock_ctx_req, 0, sizeof(struct nix_aq_enq_req)); 1246 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; 1247 lock_ctx_req.ctype = req->ctype; 1248 lock_ctx_req.op = NIX_AQ_INSTOP_LOCK; 1249 lock_ctx_req.qidx = req->qidx; 1250 err = rvu_nix_aq_enq_inst(rvu, &lock_ctx_req, NULL); 1251 if (err) 1252 dev_err(rvu->dev, 1253 "PFUNC 0x%x: Failed to lock NIX %s:%d context\n", 1254 req->hdr.pcifunc, 1255 nix_get_ctx_name(req->ctype), req->qidx); 1256 return err; 1257 } 1258 1259 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1260 struct nix_aq_enq_req *req, 1261 struct nix_aq_enq_rsp *rsp) 1262 { 1263 int err; 1264 1265 err = rvu_nix_aq_enq_inst(rvu, req, rsp); 1266 if (!err) 1267 err = nix_lf_hwctx_lockdown(rvu, req); 1268 return err; 1269 } 1270 #else 1271 1272 int rvu_mbox_handler_nix_aq_enq(struct rvu *rvu, 1273 struct nix_aq_enq_req *req, 1274 struct nix_aq_enq_rsp *rsp) 1275 { 1276 return rvu_nix_aq_enq_inst(rvu, req, rsp); 1277 } 1278 #endif 1279 /* CN10K mbox handler */ 1280 int rvu_mbox_handler_nix_cn10k_aq_enq(struct rvu *rvu, 1281 struct nix_cn10k_aq_enq_req *req, 1282 struct nix_cn10k_aq_enq_rsp *rsp) 1283 { 1284 return rvu_nix_aq_enq_inst(rvu, (struct nix_aq_enq_req *)req, 1285 (struct nix_aq_enq_rsp *)rsp); 1286 } 1287 1288 int rvu_mbox_handler_nix_hwctx_disable(struct rvu *rvu, 1289 struct hwctx_disable_req *req, 1290 struct msg_rsp *rsp) 1291 { 1292 return nix_lf_hwctx_disable(rvu, req); 1293 } 1294 1295 int rvu_mbox_handler_nix_lf_alloc(struct rvu *rvu, 1296 struct nix_lf_alloc_req *req, 1297 struct nix_lf_alloc_rsp *rsp) 1298 { 1299 int nixlf, qints, hwctx_size, intf, err, rc = 0; 1300 struct rvu_hwinfo *hw = rvu->hw; 1301 u16 pcifunc = req->hdr.pcifunc; 1302 struct rvu_block *block; 1303 struct rvu_pfvf *pfvf; 1304 u64 cfg, ctx_cfg; 1305 int blkaddr; 1306 1307 if (!req->rq_cnt || !req->sq_cnt || !req->cq_cnt) 1308 return NIX_AF_ERR_PARAM; 1309 1310 if (req->way_mask) 1311 req->way_mask &= 0xFFFF; 1312 1313 pfvf = rvu_get_pfvf(rvu, pcifunc); 1314 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1315 if (!pfvf->nixlf || blkaddr < 0) 1316 return NIX_AF_ERR_AF_LF_INVALID; 1317 1318 block = &hw->block[blkaddr]; 1319 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1320 if (nixlf < 0) 1321 return NIX_AF_ERR_AF_LF_INVALID; 1322 1323 /* Check if requested 'NIXLF <=> NPALF' mapping is valid */ 1324 if (req->npa_func) { 1325 /* If default, use 'this' NIXLF's PFFUNC */ 1326 if (req->npa_func == RVU_DEFAULT_PF_FUNC) 1327 req->npa_func = pcifunc; 1328 if (!is_pffunc_map_valid(rvu, req->npa_func, BLKTYPE_NPA)) 1329 return NIX_AF_INVAL_NPA_PF_FUNC; 1330 } 1331 1332 /* Check if requested 'NIXLF <=> SSOLF' mapping is valid */ 1333 if (req->sso_func) { 1334 /* If default, use 'this' NIXLF's PFFUNC */ 1335 if (req->sso_func == RVU_DEFAULT_PF_FUNC) 1336 req->sso_func = pcifunc; 1337 if (!is_pffunc_map_valid(rvu, req->sso_func, BLKTYPE_SSO)) 1338 return NIX_AF_INVAL_SSO_PF_FUNC; 1339 } 1340 1341 /* If RSS is being enabled, check if requested config is valid. 1342 * RSS table size should be power of two, otherwise 1343 * RSS_GRP::OFFSET + adder might go beyond that group or 1344 * won't be able to use entire table. 1345 */ 1346 if (req->rss_sz && (req->rss_sz > MAX_RSS_INDIR_TBL_SIZE || 1347 !is_power_of_2(req->rss_sz))) 1348 return NIX_AF_ERR_RSS_SIZE_INVALID; 1349 1350 if (req->rss_sz && 1351 (!req->rss_grps || req->rss_grps > MAX_RSS_GROUPS)) 1352 return NIX_AF_ERR_RSS_GRPS_INVALID; 1353 1354 /* Reset this NIX LF */ 1355 err = rvu_lf_reset(rvu, block, nixlf); 1356 if (err) { 1357 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1358 block->addr - BLKADDR_NIX0, nixlf); 1359 return NIX_AF_ERR_LF_RESET; 1360 } 1361 1362 ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3); 1363 1364 /* Alloc NIX RQ HW context memory and config the base */ 1365 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); 1366 err = qmem_alloc(rvu->dev, &pfvf->rq_ctx, req->rq_cnt, hwctx_size); 1367 if (err) 1368 goto free_mem; 1369 1370 pfvf->rq_bmap = kcalloc(req->rq_cnt, sizeof(long), GFP_KERNEL); 1371 if (!pfvf->rq_bmap) 1372 goto free_mem; 1373 1374 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_BASE(nixlf), 1375 (u64)pfvf->rq_ctx->iova); 1376 1377 /* Set caching and queue count in HW */ 1378 cfg = BIT_ULL(36) | (req->rq_cnt - 1) | req->way_mask << 20; 1379 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RQS_CFG(nixlf), cfg); 1380 1381 /* Alloc NIX SQ HW context memory and config the base */ 1382 hwctx_size = 1UL << (ctx_cfg & 0xF); 1383 err = qmem_alloc(rvu->dev, &pfvf->sq_ctx, req->sq_cnt, hwctx_size); 1384 if (err) 1385 goto free_mem; 1386 1387 pfvf->sq_bmap = kcalloc(req->sq_cnt, sizeof(long), GFP_KERNEL); 1388 if (!pfvf->sq_bmap) 1389 goto free_mem; 1390 1391 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_BASE(nixlf), 1392 (u64)pfvf->sq_ctx->iova); 1393 1394 cfg = BIT_ULL(36) | (req->sq_cnt - 1) | req->way_mask << 20; 1395 rvu_write64(rvu, blkaddr, NIX_AF_LFX_SQS_CFG(nixlf), cfg); 1396 1397 /* Alloc NIX CQ HW context memory and config the base */ 1398 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); 1399 err = qmem_alloc(rvu->dev, &pfvf->cq_ctx, req->cq_cnt, hwctx_size); 1400 if (err) 1401 goto free_mem; 1402 1403 pfvf->cq_bmap = kcalloc(req->cq_cnt, sizeof(long), GFP_KERNEL); 1404 if (!pfvf->cq_bmap) 1405 goto free_mem; 1406 1407 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_BASE(nixlf), 1408 (u64)pfvf->cq_ctx->iova); 1409 1410 cfg = BIT_ULL(36) | (req->cq_cnt - 1) | req->way_mask << 20; 1411 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CQS_CFG(nixlf), cfg); 1412 1413 /* Initialize receive side scaling (RSS) */ 1414 hwctx_size = 1UL << ((ctx_cfg >> 12) & 0xF); 1415 err = nixlf_rss_ctx_init(rvu, blkaddr, pfvf, nixlf, req->rss_sz, 1416 req->rss_grps, hwctx_size, req->way_mask, 1417 !!(req->flags & NIX_LF_RSS_TAG_LSB_AS_ADDER)); 1418 if (err) 1419 goto free_mem; 1420 1421 /* Alloc memory for CQINT's HW contexts */ 1422 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1423 qints = (cfg >> 24) & 0xFFF; 1424 hwctx_size = 1UL << ((ctx_cfg >> 24) & 0xF); 1425 err = qmem_alloc(rvu->dev, &pfvf->cq_ints_ctx, qints, hwctx_size); 1426 if (err) 1427 goto free_mem; 1428 1429 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_BASE(nixlf), 1430 (u64)pfvf->cq_ints_ctx->iova); 1431 1432 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CINTS_CFG(nixlf), 1433 BIT_ULL(36) | req->way_mask << 20); 1434 1435 /* Alloc memory for QINT's HW contexts */ 1436 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1437 qints = (cfg >> 12) & 0xFFF; 1438 hwctx_size = 1UL << ((ctx_cfg >> 20) & 0xF); 1439 err = qmem_alloc(rvu->dev, &pfvf->nix_qints_ctx, qints, hwctx_size); 1440 if (err) 1441 goto free_mem; 1442 1443 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf), 1444 (u64)pfvf->nix_qints_ctx->iova); 1445 rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_CFG(nixlf), 1446 BIT_ULL(36) | req->way_mask << 20); 1447 1448 /* Setup VLANX TPID's. 1449 * Use VLAN1 for 802.1Q 1450 * and VLAN0 for 802.1AD. 1451 */ 1452 cfg = (0x8100ULL << 16) | 0x88A8ULL; 1453 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 1454 1455 /* Enable LMTST for this NIX LF */ 1456 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG2(nixlf), BIT_ULL(0)); 1457 1458 /* Set CQE/WQE size, NPA_PF_FUNC for SQBs and also SSO_PF_FUNC */ 1459 if (req->npa_func) 1460 cfg = req->npa_func; 1461 if (req->sso_func) 1462 cfg |= (u64)req->sso_func << 16; 1463 1464 cfg |= (u64)req->xqe_sz << 33; 1465 rvu_write64(rvu, blkaddr, NIX_AF_LFX_CFG(nixlf), cfg); 1466 1467 /* Config Rx pkt length, csum checks and apad enable / disable */ 1468 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), req->rx_cfg); 1469 1470 /* Configure pkind for TX parse config */ 1471 cfg = NPC_TX_DEF_PKIND; 1472 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf), cfg); 1473 1474 intf = is_afvf(pcifunc) ? NIX_INTF_TYPE_LBK : NIX_INTF_TYPE_CGX; 1475 if (is_sdp_pfvf(pcifunc)) 1476 intf = NIX_INTF_TYPE_SDP; 1477 1478 err = nix_interface_init(rvu, pcifunc, intf, nixlf, rsp, 1479 !!(req->flags & NIX_LF_LBK_BLK_SEL)); 1480 if (err) 1481 goto free_mem; 1482 1483 /* Disable NPC entries as NIXLF's contexts are not initialized yet */ 1484 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf); 1485 1486 /* Configure RX VTAG Type 7 (strip) for vf vlan */ 1487 rvu_write64(rvu, blkaddr, 1488 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, NIX_AF_LFX_RX_VTAG_TYPE7), 1489 VTAGSIZE_T4 | VTAG_STRIP); 1490 1491 goto exit; 1492 1493 free_mem: 1494 nix_ctx_free(rvu, pfvf); 1495 rc = -ENOMEM; 1496 1497 exit: 1498 /* Set macaddr of this PF/VF */ 1499 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 1500 1501 /* set SQB size info */ 1502 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQ_CONST); 1503 rsp->sqb_size = (cfg >> 34) & 0xFFFF; 1504 rsp->rx_chan_base = pfvf->rx_chan_base; 1505 rsp->tx_chan_base = pfvf->tx_chan_base; 1506 rsp->rx_chan_cnt = pfvf->rx_chan_cnt; 1507 rsp->tx_chan_cnt = pfvf->tx_chan_cnt; 1508 rsp->lso_tsov4_idx = NIX_LSO_FORMAT_IDX_TSOV4; 1509 rsp->lso_tsov6_idx = NIX_LSO_FORMAT_IDX_TSOV6; 1510 /* Get HW supported stat count */ 1511 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 1512 rsp->lf_rx_stats = ((cfg >> 32) & 0xFF); 1513 rsp->lf_tx_stats = ((cfg >> 24) & 0xFF); 1514 /* Get count of CQ IRQs and error IRQs supported per LF */ 1515 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); 1516 rsp->qints = ((cfg >> 12) & 0xFFF); 1517 rsp->cints = ((cfg >> 24) & 0xFFF); 1518 rsp->cgx_links = hw->cgx_links; 1519 rsp->lbk_links = hw->lbk_links; 1520 rsp->sdp_links = hw->sdp_links; 1521 1522 return rc; 1523 } 1524 1525 int rvu_mbox_handler_nix_lf_free(struct rvu *rvu, struct nix_lf_free_req *req, 1526 struct msg_rsp *rsp) 1527 { 1528 struct rvu_hwinfo *hw = rvu->hw; 1529 u16 pcifunc = req->hdr.pcifunc; 1530 struct rvu_block *block; 1531 int blkaddr, nixlf, err; 1532 struct rvu_pfvf *pfvf; 1533 1534 pfvf = rvu_get_pfvf(rvu, pcifunc); 1535 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1536 if (!pfvf->nixlf || blkaddr < 0) 1537 return NIX_AF_ERR_AF_LF_INVALID; 1538 1539 block = &hw->block[blkaddr]; 1540 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 1541 if (nixlf < 0) 1542 return NIX_AF_ERR_AF_LF_INVALID; 1543 1544 if (req->flags & NIX_LF_DISABLE_FLOWS) 1545 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 1546 else 1547 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 1548 1549 /* Free any tx vtag def entries used by this NIX LF */ 1550 if (!(req->flags & NIX_LF_DONT_FREE_TX_VTAG)) 1551 nix_free_tx_vtag_entries(rvu, pcifunc); 1552 1553 nix_interface_deinit(rvu, pcifunc, nixlf); 1554 1555 /* Reset this NIX LF */ 1556 err = rvu_lf_reset(rvu, block, nixlf); 1557 if (err) { 1558 dev_err(rvu->dev, "Failed to reset NIX%d LF%d\n", 1559 block->addr - BLKADDR_NIX0, nixlf); 1560 return NIX_AF_ERR_LF_RESET; 1561 } 1562 1563 nix_ctx_free(rvu, pfvf); 1564 1565 return 0; 1566 } 1567 1568 int rvu_mbox_handler_nix_mark_format_cfg(struct rvu *rvu, 1569 struct nix_mark_format_cfg *req, 1570 struct nix_mark_format_cfg_rsp *rsp) 1571 { 1572 u16 pcifunc = req->hdr.pcifunc; 1573 struct nix_hw *nix_hw; 1574 struct rvu_pfvf *pfvf; 1575 int blkaddr, rc; 1576 u32 cfg; 1577 1578 pfvf = rvu_get_pfvf(rvu, pcifunc); 1579 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1580 if (!pfvf->nixlf || blkaddr < 0) 1581 return NIX_AF_ERR_AF_LF_INVALID; 1582 1583 nix_hw = get_nix_hw(rvu->hw, blkaddr); 1584 if (!nix_hw) 1585 return NIX_AF_ERR_INVALID_NIXBLK; 1586 1587 cfg = (((u32)req->offset & 0x7) << 16) | 1588 (((u32)req->y_mask & 0xF) << 12) | 1589 (((u32)req->y_val & 0xF) << 8) | 1590 (((u32)req->r_mask & 0xF) << 4) | ((u32)req->r_val & 0xF); 1591 1592 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfg); 1593 if (rc < 0) { 1594 dev_err(rvu->dev, "No mark_format_ctl for (pf:%d, vf:%d)", 1595 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK); 1596 return NIX_AF_ERR_MARK_CFG_FAIL; 1597 } 1598 1599 rsp->mark_format_idx = rc; 1600 return 0; 1601 } 1602 1603 /* Handle shaper update specially for few revisions */ 1604 static bool 1605 handle_txschq_shaper_update(struct rvu *rvu, int blkaddr, int nixlf, 1606 int lvl, u64 reg, u64 regval) 1607 { 1608 u64 regbase, oldval, sw_xoff = 0; 1609 u64 dbgval, md_debug0 = 0; 1610 unsigned long poll_tmo; 1611 bool rate_reg = 0; 1612 u32 schq; 1613 1614 regbase = reg & 0xFFFF; 1615 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 1616 1617 /* Check for rate register */ 1618 switch (lvl) { 1619 case NIX_TXSCH_LVL_TL1: 1620 md_debug0 = NIX_AF_TL1X_MD_DEBUG0(schq); 1621 sw_xoff = NIX_AF_TL1X_SW_XOFF(schq); 1622 1623 rate_reg = !!(regbase == NIX_AF_TL1X_CIR(0)); 1624 break; 1625 case NIX_TXSCH_LVL_TL2: 1626 md_debug0 = NIX_AF_TL2X_MD_DEBUG0(schq); 1627 sw_xoff = NIX_AF_TL2X_SW_XOFF(schq); 1628 1629 rate_reg = (regbase == NIX_AF_TL2X_CIR(0) || 1630 regbase == NIX_AF_TL2X_PIR(0)); 1631 break; 1632 case NIX_TXSCH_LVL_TL3: 1633 md_debug0 = NIX_AF_TL3X_MD_DEBUG0(schq); 1634 sw_xoff = NIX_AF_TL3X_SW_XOFF(schq); 1635 1636 rate_reg = (regbase == NIX_AF_TL3X_CIR(0) || 1637 regbase == NIX_AF_TL3X_PIR(0)); 1638 break; 1639 case NIX_TXSCH_LVL_TL4: 1640 md_debug0 = NIX_AF_TL4X_MD_DEBUG0(schq); 1641 sw_xoff = NIX_AF_TL4X_SW_XOFF(schq); 1642 1643 rate_reg = (regbase == NIX_AF_TL4X_CIR(0) || 1644 regbase == NIX_AF_TL4X_PIR(0)); 1645 break; 1646 case NIX_TXSCH_LVL_MDQ: 1647 sw_xoff = NIX_AF_MDQX_SW_XOFF(schq); 1648 rate_reg = (regbase == NIX_AF_MDQX_CIR(0) || 1649 regbase == NIX_AF_MDQX_PIR(0)); 1650 break; 1651 } 1652 1653 if (!rate_reg) 1654 return false; 1655 1656 /* Nothing special to do when state is not toggled */ 1657 oldval = rvu_read64(rvu, blkaddr, reg); 1658 if ((oldval & 0x1) == (regval & 0x1)) { 1659 rvu_write64(rvu, blkaddr, reg, regval); 1660 return true; 1661 } 1662 1663 /* PIR/CIR disable */ 1664 if (!(regval & 0x1)) { 1665 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1666 rvu_write64(rvu, blkaddr, reg, 0); 1667 udelay(4); 1668 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1669 return true; 1670 } 1671 1672 /* PIR/CIR enable */ 1673 rvu_write64(rvu, blkaddr, sw_xoff, 1); 1674 if (md_debug0) { 1675 poll_tmo = jiffies + usecs_to_jiffies(10000); 1676 /* Wait until VLD(bit32) == 1 or C_CON(bit48) == 0 */ 1677 do { 1678 if (time_after(jiffies, poll_tmo)) { 1679 dev_err(rvu->dev, 1680 "NIXLF%d: TLX%u(lvl %u) CIR/PIR enable failed\n", 1681 nixlf, schq, lvl); 1682 goto exit; 1683 } 1684 usleep_range(1, 5); 1685 dbgval = rvu_read64(rvu, blkaddr, md_debug0); 1686 } while (!(dbgval & BIT_ULL(32)) && (dbgval & BIT_ULL(48))); 1687 } 1688 rvu_write64(rvu, blkaddr, reg, regval); 1689 exit: 1690 rvu_write64(rvu, blkaddr, sw_xoff, 0); 1691 return true; 1692 } 1693 1694 static void nix_reset_tx_schedule(struct rvu *rvu, int blkaddr, 1695 int lvl, int schq) 1696 { 1697 u64 tlx_parent = 0, tlx_schedule = 0; 1698 1699 switch (lvl) { 1700 case NIX_TXSCH_LVL_TL2: 1701 tlx_parent = NIX_AF_TL2X_PARENT(schq); 1702 tlx_schedule = NIX_AF_TL2X_SCHEDULE(schq); 1703 break; 1704 case NIX_TXSCH_LVL_TL3: 1705 tlx_parent = NIX_AF_TL3X_PARENT(schq); 1706 tlx_schedule = NIX_AF_TL3X_SCHEDULE(schq); 1707 break; 1708 case NIX_TXSCH_LVL_TL4: 1709 tlx_parent = NIX_AF_TL4X_PARENT(schq); 1710 tlx_schedule = NIX_AF_TL4X_SCHEDULE(schq); 1711 break; 1712 case NIX_TXSCH_LVL_MDQ: 1713 /* no need to reset SMQ_CFG as HW clears this CSR 1714 * on SMQ flush 1715 */ 1716 tlx_parent = NIX_AF_MDQX_PARENT(schq); 1717 tlx_schedule = NIX_AF_MDQX_SCHEDULE(schq); 1718 break; 1719 default: 1720 return; 1721 } 1722 1723 if (tlx_parent) 1724 rvu_write64(rvu, blkaddr, tlx_parent, 0x0); 1725 1726 if (tlx_schedule) 1727 rvu_write64(rvu, blkaddr, tlx_schedule, 0x0); 1728 } 1729 1730 /* Disable shaping of pkts by a scheduler queue 1731 * at a given scheduler level. 1732 */ 1733 static void nix_reset_tx_shaping(struct rvu *rvu, int blkaddr, 1734 int nixlf, int lvl, int schq) 1735 { 1736 struct rvu_hwinfo *hw = rvu->hw; 1737 u64 cir_reg = 0, pir_reg = 0; 1738 u64 cfg; 1739 1740 switch (lvl) { 1741 case NIX_TXSCH_LVL_TL1: 1742 cir_reg = NIX_AF_TL1X_CIR(schq); 1743 pir_reg = 0; /* PIR not available at TL1 */ 1744 break; 1745 case NIX_TXSCH_LVL_TL2: 1746 cir_reg = NIX_AF_TL2X_CIR(schq); 1747 pir_reg = NIX_AF_TL2X_PIR(schq); 1748 break; 1749 case NIX_TXSCH_LVL_TL3: 1750 cir_reg = NIX_AF_TL3X_CIR(schq); 1751 pir_reg = NIX_AF_TL3X_PIR(schq); 1752 break; 1753 case NIX_TXSCH_LVL_TL4: 1754 cir_reg = NIX_AF_TL4X_CIR(schq); 1755 pir_reg = NIX_AF_TL4X_PIR(schq); 1756 break; 1757 case NIX_TXSCH_LVL_MDQ: 1758 cir_reg = NIX_AF_MDQX_CIR(schq); 1759 pir_reg = NIX_AF_MDQX_PIR(schq); 1760 break; 1761 } 1762 1763 /* Shaper state toggle needs wait/poll */ 1764 if (hw->cap.nix_shaper_toggle_wait) { 1765 if (cir_reg) 1766 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 1767 lvl, cir_reg, 0); 1768 if (pir_reg) 1769 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 1770 lvl, pir_reg, 0); 1771 return; 1772 } 1773 1774 if (!cir_reg) 1775 return; 1776 cfg = rvu_read64(rvu, blkaddr, cir_reg); 1777 rvu_write64(rvu, blkaddr, cir_reg, cfg & ~BIT_ULL(0)); 1778 1779 if (!pir_reg) 1780 return; 1781 cfg = rvu_read64(rvu, blkaddr, pir_reg); 1782 rvu_write64(rvu, blkaddr, pir_reg, cfg & ~BIT_ULL(0)); 1783 } 1784 1785 static void nix_reset_tx_linkcfg(struct rvu *rvu, int blkaddr, 1786 int lvl, int schq) 1787 { 1788 struct rvu_hwinfo *hw = rvu->hw; 1789 int link_level; 1790 int link; 1791 1792 if (lvl >= hw->cap.nix_tx_aggr_lvl) 1793 return; 1794 1795 /* Reset TL4's SDP link config */ 1796 if (lvl == NIX_TXSCH_LVL_TL4) 1797 rvu_write64(rvu, blkaddr, NIX_AF_TL4X_SDP_LINK_CFG(schq), 0x00); 1798 1799 link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 1800 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 1801 if (lvl != link_level) 1802 return; 1803 1804 /* Reset TL2's CGX or LBK link config */ 1805 for (link = 0; link < (hw->cgx_links + hw->lbk_links); link++) 1806 rvu_write64(rvu, blkaddr, 1807 NIX_AF_TL3_TL2X_LINKX_CFG(schq, link), 0x00); 1808 } 1809 1810 static void nix_clear_tx_xoff(struct rvu *rvu, int blkaddr, 1811 int lvl, int schq) 1812 { 1813 struct rvu_hwinfo *hw = rvu->hw; 1814 u64 reg; 1815 1816 /* Skip this if shaping is not supported */ 1817 if (!hw->cap.nix_shaping) 1818 return; 1819 1820 /* Clear level specific SW_XOFF */ 1821 switch (lvl) { 1822 case NIX_TXSCH_LVL_TL1: 1823 reg = NIX_AF_TL1X_SW_XOFF(schq); 1824 break; 1825 case NIX_TXSCH_LVL_TL2: 1826 reg = NIX_AF_TL2X_SW_XOFF(schq); 1827 break; 1828 case NIX_TXSCH_LVL_TL3: 1829 reg = NIX_AF_TL3X_SW_XOFF(schq); 1830 break; 1831 case NIX_TXSCH_LVL_TL4: 1832 reg = NIX_AF_TL4X_SW_XOFF(schq); 1833 break; 1834 case NIX_TXSCH_LVL_MDQ: 1835 reg = NIX_AF_MDQX_SW_XOFF(schq); 1836 break; 1837 default: 1838 return; 1839 } 1840 1841 rvu_write64(rvu, blkaddr, reg, 0x0); 1842 } 1843 1844 static int nix_get_tx_link(struct rvu *rvu, u16 pcifunc) 1845 { 1846 struct rvu_hwinfo *hw = rvu->hw; 1847 int pf = rvu_get_pf(pcifunc); 1848 u8 cgx_id = 0, lmac_id = 0; 1849 1850 if (is_afvf(pcifunc)) {/* LBK links */ 1851 return hw->cgx_links; 1852 } else if (is_pf_cgxmapped(rvu, pf)) { 1853 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 1854 return (cgx_id * hw->lmac_per_cgx) + lmac_id; 1855 } 1856 1857 /* SDP link */ 1858 return hw->cgx_links + hw->lbk_links; 1859 } 1860 1861 static void nix_get_txschq_range(struct rvu *rvu, u16 pcifunc, 1862 int link, int *start, int *end) 1863 { 1864 struct rvu_hwinfo *hw = rvu->hw; 1865 int pf = rvu_get_pf(pcifunc); 1866 1867 if (is_afvf(pcifunc)) { /* LBK links */ 1868 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 1869 *end = *start + hw->cap.nix_txsch_per_lbk_lmac; 1870 } else if (is_pf_cgxmapped(rvu, pf)) { /* CGX links */ 1871 *start = hw->cap.nix_txsch_per_cgx_lmac * link; 1872 *end = *start + hw->cap.nix_txsch_per_cgx_lmac; 1873 } else { /* SDP link */ 1874 *start = (hw->cap.nix_txsch_per_cgx_lmac * hw->cgx_links) + 1875 (hw->cap.nix_txsch_per_lbk_lmac * hw->lbk_links); 1876 *end = *start + hw->cap.nix_txsch_per_sdp_lmac; 1877 } 1878 } 1879 1880 static int nix_check_txschq_alloc_req(struct rvu *rvu, int lvl, u16 pcifunc, 1881 struct nix_hw *nix_hw, 1882 struct nix_txsch_alloc_req *req) 1883 { 1884 struct rvu_hwinfo *hw = rvu->hw; 1885 int schq, req_schq, free_cnt; 1886 struct nix_txsch *txsch; 1887 int link, start, end; 1888 1889 txsch = &nix_hw->txsch[lvl]; 1890 req_schq = req->schq_contig[lvl] + req->schq[lvl]; 1891 1892 if (!req_schq) 1893 return 0; 1894 1895 link = nix_get_tx_link(rvu, pcifunc); 1896 1897 /* For traffic aggregating scheduler level, one queue is enough */ 1898 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 1899 if (req_schq != 1) 1900 return NIX_AF_ERR_TLX_ALLOC_FAIL; 1901 return 0; 1902 } 1903 1904 /* Get free SCHQ count and check if request can be accomodated */ 1905 if (hw->cap.nix_fixed_txschq_mapping) { 1906 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 1907 schq = start + (pcifunc & RVU_PFVF_FUNC_MASK); 1908 if (end <= txsch->schq.max && schq < end && 1909 !test_bit(schq, txsch->schq.bmap)) 1910 free_cnt = 1; 1911 else 1912 free_cnt = 0; 1913 } else { 1914 free_cnt = rvu_rsrc_free_count(&txsch->schq); 1915 } 1916 1917 if (free_cnt < req_schq || req_schq > MAX_TXSCHQ_PER_FUNC) 1918 return NIX_AF_ERR_TLX_ALLOC_FAIL; 1919 1920 /* If contiguous queues are needed, check for availability */ 1921 if (!hw->cap.nix_fixed_txschq_mapping && req->schq_contig[lvl] && 1922 !rvu_rsrc_check_contig(&txsch->schq, req->schq_contig[lvl])) 1923 return NIX_AF_ERR_TLX_ALLOC_FAIL; 1924 1925 return 0; 1926 } 1927 1928 static void nix_txsch_alloc(struct rvu *rvu, struct nix_txsch *txsch, 1929 struct nix_txsch_alloc_rsp *rsp, 1930 int lvl, int start, int end) 1931 { 1932 struct rvu_hwinfo *hw = rvu->hw; 1933 u16 pcifunc = rsp->hdr.pcifunc; 1934 int idx, schq; 1935 1936 /* For traffic aggregating levels, queue alloc is based 1937 * on transmit link to which PF_FUNC is mapped to. 1938 */ 1939 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 1940 /* A single TL queue is allocated */ 1941 if (rsp->schq_contig[lvl]) { 1942 rsp->schq_contig[lvl] = 1; 1943 rsp->schq_contig_list[lvl][0] = start; 1944 } 1945 1946 /* Both contig and non-contig reqs doesn't make sense here */ 1947 if (rsp->schq_contig[lvl]) 1948 rsp->schq[lvl] = 0; 1949 1950 if (rsp->schq[lvl]) { 1951 rsp->schq[lvl] = 1; 1952 rsp->schq_list[lvl][0] = start; 1953 } 1954 return; 1955 } 1956 1957 /* Adjust the queue request count if HW supports 1958 * only one queue per level configuration. 1959 */ 1960 if (hw->cap.nix_fixed_txschq_mapping) { 1961 idx = pcifunc & RVU_PFVF_FUNC_MASK; 1962 schq = start + idx; 1963 if (idx >= (end - start) || test_bit(schq, txsch->schq.bmap)) { 1964 rsp->schq_contig[lvl] = 0; 1965 rsp->schq[lvl] = 0; 1966 return; 1967 } 1968 1969 if (rsp->schq_contig[lvl]) { 1970 rsp->schq_contig[lvl] = 1; 1971 set_bit(schq, txsch->schq.bmap); 1972 rsp->schq_contig_list[lvl][0] = schq; 1973 rsp->schq[lvl] = 0; 1974 } else if (rsp->schq[lvl]) { 1975 rsp->schq[lvl] = 1; 1976 set_bit(schq, txsch->schq.bmap); 1977 rsp->schq_list[lvl][0] = schq; 1978 } 1979 return; 1980 } 1981 1982 /* Allocate contiguous queue indices requesty first */ 1983 if (rsp->schq_contig[lvl]) { 1984 schq = bitmap_find_next_zero_area(txsch->schq.bmap, 1985 txsch->schq.max, start, 1986 rsp->schq_contig[lvl], 0); 1987 if (schq >= end) 1988 rsp->schq_contig[lvl] = 0; 1989 for (idx = 0; idx < rsp->schq_contig[lvl]; idx++) { 1990 set_bit(schq, txsch->schq.bmap); 1991 rsp->schq_contig_list[lvl][idx] = schq; 1992 schq++; 1993 } 1994 } 1995 1996 /* Allocate non-contiguous queue indices */ 1997 if (rsp->schq[lvl]) { 1998 idx = 0; 1999 for (schq = start; schq < end; schq++) { 2000 if (!test_bit(schq, txsch->schq.bmap)) { 2001 set_bit(schq, txsch->schq.bmap); 2002 rsp->schq_list[lvl][idx++] = schq; 2003 } 2004 if (idx == rsp->schq[lvl]) 2005 break; 2006 } 2007 /* Update how many were allocated */ 2008 rsp->schq[lvl] = idx; 2009 } 2010 } 2011 2012 int rvu_mbox_handler_nix_txsch_alloc(struct rvu *rvu, 2013 struct nix_txsch_alloc_req *req, 2014 struct nix_txsch_alloc_rsp *rsp) 2015 { 2016 struct rvu_hwinfo *hw = rvu->hw; 2017 u16 pcifunc = req->hdr.pcifunc; 2018 int link, blkaddr, rc = 0; 2019 int lvl, idx, start, end; 2020 struct nix_txsch *txsch; 2021 struct nix_hw *nix_hw; 2022 u32 *pfvf_map; 2023 int nixlf; 2024 u16 schq; 2025 2026 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2027 if (rc) 2028 return rc; 2029 2030 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2031 if (!nix_hw) 2032 return NIX_AF_ERR_INVALID_NIXBLK; 2033 2034 mutex_lock(&rvu->rsrc_lock); 2035 2036 /* Check if request is valid as per HW capabilities 2037 * and can be accomodated. 2038 */ 2039 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2040 rc = nix_check_txschq_alloc_req(rvu, lvl, pcifunc, nix_hw, req); 2041 if (rc) 2042 goto err; 2043 } 2044 2045 /* Allocate requested Tx scheduler queues */ 2046 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2047 txsch = &nix_hw->txsch[lvl]; 2048 pfvf_map = txsch->pfvf_map; 2049 2050 if (!req->schq[lvl] && !req->schq_contig[lvl]) 2051 continue; 2052 2053 rsp->schq[lvl] = req->schq[lvl]; 2054 rsp->schq_contig[lvl] = req->schq_contig[lvl]; 2055 2056 link = nix_get_tx_link(rvu, pcifunc); 2057 2058 if (lvl >= hw->cap.nix_tx_aggr_lvl) { 2059 start = link; 2060 end = link; 2061 } else if (hw->cap.nix_fixed_txschq_mapping) { 2062 nix_get_txschq_range(rvu, pcifunc, link, &start, &end); 2063 } else { 2064 start = 0; 2065 end = txsch->schq.max; 2066 } 2067 2068 nix_txsch_alloc(rvu, txsch, rsp, lvl, start, end); 2069 2070 /* Reset queue config */ 2071 for (idx = 0; idx < req->schq_contig[lvl]; idx++) { 2072 schq = rsp->schq_contig_list[lvl][idx]; 2073 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2074 NIX_TXSCHQ_CFG_DONE)) 2075 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2076 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2077 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2078 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2079 } 2080 2081 for (idx = 0; idx < req->schq[lvl]; idx++) { 2082 schq = rsp->schq_list[lvl][idx]; 2083 if (!(TXSCH_MAP_FLAGS(pfvf_map[schq]) & 2084 NIX_TXSCHQ_CFG_DONE)) 2085 pfvf_map[schq] = TXSCH_MAP(pcifunc, 0); 2086 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2087 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2088 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2089 } 2090 } 2091 2092 rsp->aggr_level = hw->cap.nix_tx_aggr_lvl; 2093 rsp->aggr_lvl_rr_prio = TXSCH_TL1_DFLT_RR_PRIO; 2094 rsp->link_cfg_lvl = rvu_read64(rvu, blkaddr, 2095 NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? 2096 NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; 2097 goto exit; 2098 err: 2099 rc = NIX_AF_ERR_TLX_ALLOC_FAIL; 2100 exit: 2101 mutex_unlock(&rvu->rsrc_lock); 2102 return rc; 2103 } 2104 2105 static int nix_smq_flush(struct rvu *rvu, int blkaddr, 2106 int smq, u16 pcifunc, int nixlf) 2107 { 2108 int pf = rvu_get_pf(pcifunc); 2109 u8 cgx_id = 0, lmac_id = 0; 2110 int err, restore_tx_en = 0; 2111 u64 cfg; 2112 2113 if (!is_rvu_otx2(rvu)) { 2114 /* Skip SMQ flush if pkt count is zero */ 2115 cfg = rvu_read64(rvu, blkaddr, NIX_AF_MDQX_IN_MD_COUNT(smq)); 2116 if (!cfg) 2117 return 0; 2118 } 2119 2120 /* enable cgx tx if disabled */ 2121 if (is_pf_cgxmapped(rvu, pf)) { 2122 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 2123 restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), 2124 lmac_id, true); 2125 } 2126 2127 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); 2128 /* Do SMQ flush and set enqueue xoff */ 2129 cfg |= BIT_ULL(50) | BIT_ULL(49); 2130 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq), cfg); 2131 2132 /* Disable backpressure from physical link, 2133 * otherwise SMQ flush may stall. 2134 */ 2135 rvu_cgx_enadis_rx_bp(rvu, pf, false); 2136 2137 /* Wait for flush to complete */ 2138 err = rvu_poll_reg(rvu, blkaddr, 2139 NIX_AF_SMQX_CFG(smq), BIT_ULL(49), true); 2140 if (err) 2141 dev_err(rvu->dev, 2142 "NIXLF%d: SMQ%d flush failed\n", nixlf, smq); 2143 2144 rvu_cgx_enadis_rx_bp(rvu, pf, true); 2145 /* restore cgx tx state */ 2146 if (restore_tx_en) 2147 rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); 2148 return err; 2149 } 2150 2151 static int nix_txschq_free(struct rvu *rvu, u16 pcifunc) 2152 { 2153 int blkaddr, nixlf, lvl, schq, err; 2154 struct rvu_hwinfo *hw = rvu->hw; 2155 struct nix_txsch *txsch; 2156 struct nix_hw *nix_hw; 2157 u16 map_func; 2158 2159 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2160 if (blkaddr < 0) 2161 return NIX_AF_ERR_AF_LF_INVALID; 2162 2163 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2164 if (!nix_hw) 2165 return NIX_AF_ERR_INVALID_NIXBLK; 2166 2167 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2168 if (nixlf < 0) 2169 return NIX_AF_ERR_AF_LF_INVALID; 2170 2171 /* Disable TL2/3 queue links and all XOFF's before SMQ flush*/ 2172 mutex_lock(&rvu->rsrc_lock); 2173 for (lvl = NIX_TXSCH_LVL_MDQ; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2174 txsch = &nix_hw->txsch[lvl]; 2175 2176 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2177 continue; 2178 2179 for (schq = 0; schq < txsch->schq.max; schq++) { 2180 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2181 continue; 2182 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2183 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2184 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2185 } 2186 } 2187 nix_clear_tx_xoff(rvu, blkaddr, NIX_TXSCH_LVL_TL1, 2188 nix_get_tx_link(rvu, pcifunc)); 2189 2190 /* On PF cleanup, clear cfg done flag as 2191 * PF would have changed default config. 2192 */ 2193 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { 2194 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL1]; 2195 schq = nix_get_tx_link(rvu, pcifunc); 2196 /* Do not clear pcifunc in txsch->pfvf_map[schq] because 2197 * VF might be using this TL1 queue 2198 */ 2199 map_func = TXSCH_MAP_FUNC(txsch->pfvf_map[schq]); 2200 txsch->pfvf_map[schq] = TXSCH_SET_FLAG(map_func, 0x0); 2201 } 2202 2203 /* Flush SMQs */ 2204 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ]; 2205 for (schq = 0; schq < txsch->schq.max; schq++) { 2206 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2207 continue; 2208 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2209 } 2210 2211 /* Now free scheduler queues to free pool */ 2212 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 2213 /* TLs above aggregation level are shared across all PF 2214 * and it's VFs, hence skip freeing them. 2215 */ 2216 if (lvl >= hw->cap.nix_tx_aggr_lvl) 2217 continue; 2218 2219 txsch = &nix_hw->txsch[lvl]; 2220 for (schq = 0; schq < txsch->schq.max; schq++) { 2221 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2222 continue; 2223 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2224 rvu_free_rsrc(&txsch->schq, schq); 2225 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2226 } 2227 } 2228 mutex_unlock(&rvu->rsrc_lock); 2229 2230 /* Sync cached info for this LF in NDC-TX to LLC/DRAM */ 2231 rvu_write64(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12) | nixlf); 2232 err = rvu_poll_reg(rvu, blkaddr, NIX_AF_NDC_TX_SYNC, BIT_ULL(12), true); 2233 if (err) 2234 dev_err(rvu->dev, "NDC-TX sync failed for NIXLF %d\n", nixlf); 2235 2236 return 0; 2237 } 2238 2239 static int nix_txschq_free_one(struct rvu *rvu, 2240 struct nix_txsch_free_req *req) 2241 { 2242 struct rvu_hwinfo *hw = rvu->hw; 2243 u16 pcifunc = req->hdr.pcifunc; 2244 int lvl, schq, nixlf, blkaddr; 2245 struct nix_txsch *txsch; 2246 struct nix_hw *nix_hw; 2247 u32 *pfvf_map; 2248 int rc; 2249 2250 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2251 if (blkaddr < 0) 2252 return NIX_AF_ERR_AF_LF_INVALID; 2253 2254 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2255 if (!nix_hw) 2256 return NIX_AF_ERR_INVALID_NIXBLK; 2257 2258 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2259 if (nixlf < 0) 2260 return NIX_AF_ERR_AF_LF_INVALID; 2261 2262 lvl = req->schq_lvl; 2263 schq = req->schq; 2264 txsch = &nix_hw->txsch[lvl]; 2265 2266 if (lvl >= hw->cap.nix_tx_aggr_lvl || schq >= txsch->schq.max) 2267 return 0; 2268 2269 pfvf_map = txsch->pfvf_map; 2270 mutex_lock(&rvu->rsrc_lock); 2271 2272 if (TXSCH_MAP_FUNC(pfvf_map[schq]) != pcifunc) { 2273 rc = NIX_AF_ERR_TLX_INVALID; 2274 goto err; 2275 } 2276 2277 /* Clear SW_XOFF of this resource only. 2278 * For SMQ level, all path XOFF's 2279 * need to be made clear by user 2280 */ 2281 nix_clear_tx_xoff(rvu, blkaddr, lvl, schq); 2282 2283 nix_reset_tx_linkcfg(rvu, blkaddr, lvl, schq); 2284 nix_reset_tx_shaping(rvu, blkaddr, nixlf, lvl, schq); 2285 2286 /* Flush if it is a SMQ. Onus of disabling 2287 * TL2/3 queue links before SMQ flush is on user 2288 */ 2289 if (lvl == NIX_TXSCH_LVL_SMQ && 2290 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf)) { 2291 rc = NIX_AF_SMQ_FLUSH_FAILED; 2292 goto err; 2293 } 2294 2295 nix_reset_tx_schedule(rvu, blkaddr, lvl, schq); 2296 2297 /* Free the resource */ 2298 rvu_free_rsrc(&txsch->schq, schq); 2299 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 2300 mutex_unlock(&rvu->rsrc_lock); 2301 return 0; 2302 err: 2303 mutex_unlock(&rvu->rsrc_lock); 2304 return rc; 2305 } 2306 2307 int rvu_mbox_handler_nix_txsch_free(struct rvu *rvu, 2308 struct nix_txsch_free_req *req, 2309 struct msg_rsp *rsp) 2310 { 2311 if (req->flags & TXSCHQ_FREE_ALL) 2312 return nix_txschq_free(rvu, req->hdr.pcifunc); 2313 else 2314 return nix_txschq_free_one(rvu, req); 2315 } 2316 2317 static bool is_txschq_hierarchy_valid(struct rvu *rvu, u16 pcifunc, int blkaddr, 2318 int lvl, u64 reg, u64 regval) 2319 { 2320 u64 regbase = reg & 0xFFFF; 2321 u16 schq, parent; 2322 2323 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, lvl, reg)) 2324 return false; 2325 2326 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2327 /* Check if this schq belongs to this PF/VF or not */ 2328 if (!is_valid_txschq(rvu, blkaddr, lvl, pcifunc, schq)) 2329 return false; 2330 2331 parent = (regval >> 16) & 0x1FF; 2332 /* Validate MDQ's TL4 parent */ 2333 if (regbase == NIX_AF_MDQX_PARENT(0) && 2334 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL4, pcifunc, parent)) 2335 return false; 2336 2337 /* Validate TL4's TL3 parent */ 2338 if (regbase == NIX_AF_TL4X_PARENT(0) && 2339 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL3, pcifunc, parent)) 2340 return false; 2341 2342 /* Validate TL3's TL2 parent */ 2343 if (regbase == NIX_AF_TL3X_PARENT(0) && 2344 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL2, pcifunc, parent)) 2345 return false; 2346 2347 /* Validate TL2's TL1 parent */ 2348 if (regbase == NIX_AF_TL2X_PARENT(0) && 2349 !is_valid_txschq(rvu, blkaddr, NIX_TXSCH_LVL_TL1, pcifunc, parent)) 2350 return false; 2351 2352 return true; 2353 } 2354 2355 static bool is_txschq_shaping_valid(struct rvu_hwinfo *hw, int lvl, u64 reg) 2356 { 2357 u64 regbase; 2358 2359 if (hw->cap.nix_shaping) 2360 return true; 2361 2362 /* If shaping and coloring is not supported, then 2363 * *_CIR and *_PIR registers should not be configured. 2364 */ 2365 regbase = reg & 0xFFFF; 2366 2367 switch (lvl) { 2368 case NIX_TXSCH_LVL_TL1: 2369 if (regbase == NIX_AF_TL1X_CIR(0)) 2370 return false; 2371 break; 2372 case NIX_TXSCH_LVL_TL2: 2373 if (regbase == NIX_AF_TL2X_CIR(0) || 2374 regbase == NIX_AF_TL2X_PIR(0)) 2375 return false; 2376 break; 2377 case NIX_TXSCH_LVL_TL3: 2378 if (regbase == NIX_AF_TL3X_CIR(0) || 2379 regbase == NIX_AF_TL3X_PIR(0)) 2380 return false; 2381 break; 2382 case NIX_TXSCH_LVL_TL4: 2383 if (regbase == NIX_AF_TL4X_CIR(0) || 2384 regbase == NIX_AF_TL4X_PIR(0)) 2385 return false; 2386 break; 2387 case NIX_TXSCH_LVL_MDQ: 2388 if (regbase == NIX_AF_MDQX_CIR(0) || 2389 regbase == NIX_AF_MDQX_PIR(0)) 2390 return false; 2391 break; 2392 } 2393 return true; 2394 } 2395 2396 static void nix_tl1_default_cfg(struct rvu *rvu, struct nix_hw *nix_hw, 2397 u16 pcifunc, int blkaddr) 2398 { 2399 u32 *pfvf_map; 2400 int schq; 2401 2402 schq = nix_get_tx_link(rvu, pcifunc); 2403 pfvf_map = nix_hw->txsch[NIX_TXSCH_LVL_TL1].pfvf_map; 2404 /* Skip if PF has already done the config */ 2405 if (TXSCH_MAP_FLAGS(pfvf_map[schq]) & NIX_TXSCHQ_CFG_DONE) 2406 return; 2407 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_TOPOLOGY(schq), 2408 (TXSCH_TL1_DFLT_RR_PRIO << 1)); 2409 2410 /* On OcteonTx2 the config was in bytes and newer silcons 2411 * it's changed to weight. 2412 */ 2413 if (!rvu->hw->cap.nix_common_dwrr_mtu) 2414 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2415 TXSCH_TL1_DFLT_RR_QTM); 2416 else 2417 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SCHEDULE(schq), 2418 CN10K_MAX_DWRR_WEIGHT); 2419 2420 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_CIR(schq), 0x00); 2421 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], NIX_TXSCHQ_CFG_DONE); 2422 } 2423 2424 /* Register offset - [15:0] 2425 * Scheduler Queue number - [25:16] 2426 */ 2427 #define NIX_TX_SCHQ_MASK GENMASK_ULL(25, 0) 2428 2429 static int nix_txschq_cfg_read(struct rvu *rvu, struct nix_hw *nix_hw, 2430 int blkaddr, struct nix_txschq_config *req, 2431 struct nix_txschq_config *rsp) 2432 { 2433 u16 pcifunc = req->hdr.pcifunc; 2434 int idx, schq; 2435 u64 reg; 2436 2437 for (idx = 0; idx < req->num_regs; idx++) { 2438 reg = req->reg[idx]; 2439 reg &= NIX_TX_SCHQ_MASK; 2440 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2441 if (!rvu_check_valid_reg(TXSCHQ_HWREGMAP, req->lvl, reg) || 2442 !is_valid_txschq(rvu, blkaddr, req->lvl, pcifunc, schq)) 2443 return NIX_AF_INVAL_TXSCHQ_CFG; 2444 rsp->regval[idx] = rvu_read64(rvu, blkaddr, reg); 2445 } 2446 rsp->lvl = req->lvl; 2447 rsp->num_regs = req->num_regs; 2448 return 0; 2449 } 2450 2451 static void rvu_nix_tx_tl2_cfg(struct rvu *rvu, int blkaddr, 2452 u16 pcifunc, struct nix_txsch *txsch) 2453 { 2454 struct rvu_hwinfo *hw = rvu->hw; 2455 int lbk_link_start, lbk_links; 2456 u8 pf = rvu_get_pf(pcifunc); 2457 int schq; 2458 2459 if (!is_pf_cgxmapped(rvu, pf)) 2460 return; 2461 2462 lbk_link_start = hw->cgx_links; 2463 2464 for (schq = 0; schq < txsch->schq.max; schq++) { 2465 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 2466 continue; 2467 /* Enable all LBK links with channel 63 by default so that 2468 * packets can be sent to LBK with a NPC TX MCAM rule 2469 */ 2470 lbk_links = hw->lbk_links; 2471 while (lbk_links--) 2472 rvu_write64(rvu, blkaddr, 2473 NIX_AF_TL3_TL2X_LINKX_CFG(schq, 2474 lbk_link_start + 2475 lbk_links), 2476 BIT_ULL(12) | RVU_SWITCH_LBK_CHAN); 2477 } 2478 } 2479 2480 int rvu_mbox_handler_nix_txschq_cfg(struct rvu *rvu, 2481 struct nix_txschq_config *req, 2482 struct nix_txschq_config *rsp) 2483 { 2484 u64 reg, val, regval, schq_regbase, val_mask; 2485 struct rvu_hwinfo *hw = rvu->hw; 2486 u16 pcifunc = req->hdr.pcifunc; 2487 struct nix_txsch *txsch; 2488 struct nix_hw *nix_hw; 2489 int blkaddr, idx, err; 2490 int nixlf, schq; 2491 u32 *pfvf_map; 2492 2493 if (req->lvl >= NIX_TXSCH_LVL_CNT || 2494 req->num_regs > MAX_REGS_PER_MBOX_MSG) 2495 return NIX_AF_INVAL_TXSCHQ_CFG; 2496 2497 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2498 if (err) 2499 return err; 2500 2501 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2502 if (!nix_hw) 2503 return NIX_AF_ERR_INVALID_NIXBLK; 2504 2505 if (req->read) 2506 return nix_txschq_cfg_read(rvu, nix_hw, blkaddr, req, rsp); 2507 2508 txsch = &nix_hw->txsch[req->lvl]; 2509 pfvf_map = txsch->pfvf_map; 2510 2511 if (req->lvl >= hw->cap.nix_tx_aggr_lvl && 2512 pcifunc & RVU_PFVF_FUNC_MASK) { 2513 mutex_lock(&rvu->rsrc_lock); 2514 if (req->lvl == NIX_TXSCH_LVL_TL1) 2515 nix_tl1_default_cfg(rvu, nix_hw, pcifunc, blkaddr); 2516 mutex_unlock(&rvu->rsrc_lock); 2517 return 0; 2518 } 2519 2520 for (idx = 0; idx < req->num_regs; idx++) { 2521 reg = req->reg[idx]; 2522 reg &= NIX_TX_SCHQ_MASK; 2523 regval = req->regval[idx]; 2524 schq_regbase = reg & 0xFFFF; 2525 val_mask = req->regval_mask[idx]; 2526 2527 if (!is_txschq_hierarchy_valid(rvu, pcifunc, blkaddr, 2528 txsch->lvl, reg, regval)) 2529 return NIX_AF_INVAL_TXSCHQ_CFG; 2530 2531 /* Check if shaping and coloring is supported */ 2532 if (!is_txschq_shaping_valid(hw, req->lvl, reg)) 2533 continue; 2534 2535 val = rvu_read64(rvu, blkaddr, reg); 2536 regval = (val & val_mask) | (regval & ~val_mask); 2537 2538 /* Handle shaping state toggle specially */ 2539 if (hw->cap.nix_shaper_toggle_wait && 2540 handle_txschq_shaper_update(rvu, blkaddr, nixlf, 2541 req->lvl, reg, regval)) 2542 continue; 2543 2544 /* Replace PF/VF visible NIXLF slot with HW NIXLF id */ 2545 if (schq_regbase == NIX_AF_SMQX_CFG(0)) { 2546 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], 2547 pcifunc, 0); 2548 regval &= ~(0x7FULL << 24); 2549 regval |= ((u64)nixlf << 24); 2550 } 2551 2552 /* Clear 'BP_ENA' config, if it's not allowed */ 2553 if (!hw->cap.nix_tx_link_bp) { 2554 if (schq_regbase == NIX_AF_TL4X_SDP_LINK_CFG(0) || 2555 (schq_regbase & 0xFF00) == 2556 NIX_AF_TL3_TL2X_LINKX_CFG(0, 0)) 2557 regval &= ~BIT_ULL(13); 2558 } 2559 2560 /* Mark config as done for TL1 by PF */ 2561 if (schq_regbase >= NIX_AF_TL1X_SCHEDULE(0) && 2562 schq_regbase <= NIX_AF_TL1X_GREEN_BYTES(0)) { 2563 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2564 mutex_lock(&rvu->rsrc_lock); 2565 pfvf_map[schq] = TXSCH_SET_FLAG(pfvf_map[schq], 2566 NIX_TXSCHQ_CFG_DONE); 2567 mutex_unlock(&rvu->rsrc_lock); 2568 } 2569 2570 /* SMQ flush is special hence split register writes such 2571 * that flush first and write rest of the bits later. 2572 */ 2573 if (schq_regbase == NIX_AF_SMQX_CFG(0) && 2574 (regval & BIT_ULL(49))) { 2575 schq = TXSCHQ_IDX(reg, TXSCHQ_IDX_SHIFT); 2576 nix_smq_flush(rvu, blkaddr, schq, pcifunc, nixlf); 2577 regval &= ~BIT_ULL(49); 2578 } 2579 rvu_write64(rvu, blkaddr, reg, regval); 2580 } 2581 2582 rvu_nix_tx_tl2_cfg(rvu, blkaddr, pcifunc, 2583 &nix_hw->txsch[NIX_TXSCH_LVL_TL2]); 2584 return 0; 2585 } 2586 2587 static int nix_rx_vtag_cfg(struct rvu *rvu, int nixlf, int blkaddr, 2588 struct nix_vtag_config *req) 2589 { 2590 u64 regval = req->vtag_size; 2591 2592 if (req->rx.vtag_type > NIX_AF_LFX_RX_VTAG_TYPE7 || 2593 req->vtag_size > VTAGSIZE_T8) 2594 return -EINVAL; 2595 2596 /* RX VTAG Type 7 reserved for vf vlan */ 2597 if (req->rx.vtag_type == NIX_AF_LFX_RX_VTAG_TYPE7) 2598 return NIX_AF_ERR_RX_VTAG_INUSE; 2599 2600 if (req->rx.capture_vtag) 2601 regval |= BIT_ULL(5); 2602 if (req->rx.strip_vtag) 2603 regval |= BIT_ULL(4); 2604 2605 rvu_write64(rvu, blkaddr, 2606 NIX_AF_LFX_RX_VTAG_TYPEX(nixlf, req->rx.vtag_type), regval); 2607 return 0; 2608 } 2609 2610 static int nix_tx_vtag_free(struct rvu *rvu, int blkaddr, 2611 u16 pcifunc, int index) 2612 { 2613 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 2614 struct nix_txvlan *vlan; 2615 2616 if (!nix_hw) 2617 return NIX_AF_ERR_INVALID_NIXBLK; 2618 2619 vlan = &nix_hw->txvlan; 2620 if (vlan->entry2pfvf_map[index] != pcifunc) 2621 return NIX_AF_ERR_PARAM; 2622 2623 rvu_write64(rvu, blkaddr, 2624 NIX_AF_TX_VTAG_DEFX_DATA(index), 0x0ull); 2625 rvu_write64(rvu, blkaddr, 2626 NIX_AF_TX_VTAG_DEFX_CTL(index), 0x0ull); 2627 2628 vlan->entry2pfvf_map[index] = 0; 2629 rvu_free_rsrc(&vlan->rsrc, index); 2630 2631 return 0; 2632 } 2633 2634 static void nix_free_tx_vtag_entries(struct rvu *rvu, u16 pcifunc) 2635 { 2636 struct nix_txvlan *vlan; 2637 struct nix_hw *nix_hw; 2638 int index, blkaddr; 2639 2640 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2641 if (blkaddr < 0) 2642 return; 2643 2644 nix_hw = get_nix_hw(rvu->hw, blkaddr); 2645 if (!nix_hw) 2646 return; 2647 2648 vlan = &nix_hw->txvlan; 2649 2650 mutex_lock(&vlan->rsrc_lock); 2651 /* Scan all the entries and free the ones mapped to 'pcifunc' */ 2652 for (index = 0; index < vlan->rsrc.max; index++) { 2653 if (vlan->entry2pfvf_map[index] == pcifunc) 2654 nix_tx_vtag_free(rvu, blkaddr, pcifunc, index); 2655 } 2656 mutex_unlock(&vlan->rsrc_lock); 2657 } 2658 2659 static int nix_tx_vtag_alloc(struct rvu *rvu, int blkaddr, 2660 u64 vtag, u8 size) 2661 { 2662 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 2663 struct nix_txvlan *vlan; 2664 u64 regval; 2665 int index; 2666 2667 if (!nix_hw) 2668 return NIX_AF_ERR_INVALID_NIXBLK; 2669 2670 vlan = &nix_hw->txvlan; 2671 2672 mutex_lock(&vlan->rsrc_lock); 2673 2674 index = rvu_alloc_rsrc(&vlan->rsrc); 2675 if (index < 0) { 2676 mutex_unlock(&vlan->rsrc_lock); 2677 return index; 2678 } 2679 2680 mutex_unlock(&vlan->rsrc_lock); 2681 2682 regval = size ? vtag : vtag << 32; 2683 2684 rvu_write64(rvu, blkaddr, 2685 NIX_AF_TX_VTAG_DEFX_DATA(index), regval); 2686 rvu_write64(rvu, blkaddr, 2687 NIX_AF_TX_VTAG_DEFX_CTL(index), size); 2688 2689 return index; 2690 } 2691 2692 static int nix_tx_vtag_decfg(struct rvu *rvu, int blkaddr, 2693 struct nix_vtag_config *req) 2694 { 2695 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 2696 u16 pcifunc = req->hdr.pcifunc; 2697 int idx0 = req->tx.vtag0_idx; 2698 int idx1 = req->tx.vtag1_idx; 2699 struct nix_txvlan *vlan; 2700 int err = 0; 2701 2702 if (!nix_hw) 2703 return NIX_AF_ERR_INVALID_NIXBLK; 2704 2705 vlan = &nix_hw->txvlan; 2706 if (req->tx.free_vtag0 && req->tx.free_vtag1) 2707 if (vlan->entry2pfvf_map[idx0] != pcifunc || 2708 vlan->entry2pfvf_map[idx1] != pcifunc) 2709 return NIX_AF_ERR_PARAM; 2710 2711 mutex_lock(&vlan->rsrc_lock); 2712 2713 if (req->tx.free_vtag0) { 2714 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx0); 2715 if (err) 2716 goto exit; 2717 } 2718 2719 if (req->tx.free_vtag1) 2720 err = nix_tx_vtag_free(rvu, blkaddr, pcifunc, idx1); 2721 2722 exit: 2723 mutex_unlock(&vlan->rsrc_lock); 2724 return err; 2725 } 2726 2727 static int nix_tx_vtag_cfg(struct rvu *rvu, int blkaddr, 2728 struct nix_vtag_config *req, 2729 struct nix_vtag_config_rsp *rsp) 2730 { 2731 struct nix_hw *nix_hw = get_nix_hw(rvu->hw, blkaddr); 2732 struct nix_txvlan *vlan; 2733 u16 pcifunc = req->hdr.pcifunc; 2734 2735 if (!nix_hw) 2736 return NIX_AF_ERR_INVALID_NIXBLK; 2737 2738 vlan = &nix_hw->txvlan; 2739 if (req->tx.cfg_vtag0) { 2740 rsp->vtag0_idx = 2741 nix_tx_vtag_alloc(rvu, blkaddr, 2742 req->tx.vtag0, req->vtag_size); 2743 2744 if (rsp->vtag0_idx < 0) 2745 return NIX_AF_ERR_TX_VTAG_NOSPC; 2746 2747 vlan->entry2pfvf_map[rsp->vtag0_idx] = pcifunc; 2748 } 2749 2750 if (req->tx.cfg_vtag1) { 2751 rsp->vtag1_idx = 2752 nix_tx_vtag_alloc(rvu, blkaddr, 2753 req->tx.vtag1, req->vtag_size); 2754 2755 if (rsp->vtag1_idx < 0) 2756 goto err_free; 2757 2758 vlan->entry2pfvf_map[rsp->vtag1_idx] = pcifunc; 2759 } 2760 2761 return 0; 2762 2763 err_free: 2764 if (req->tx.cfg_vtag0) 2765 nix_tx_vtag_free(rvu, blkaddr, pcifunc, rsp->vtag0_idx); 2766 2767 return NIX_AF_ERR_TX_VTAG_NOSPC; 2768 } 2769 2770 int rvu_mbox_handler_nix_vtag_cfg(struct rvu *rvu, 2771 struct nix_vtag_config *req, 2772 struct nix_vtag_config_rsp *rsp) 2773 { 2774 u16 pcifunc = req->hdr.pcifunc; 2775 int blkaddr, nixlf, err; 2776 2777 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 2778 if (err) 2779 return err; 2780 2781 if (req->cfg_type) { 2782 /* rx vtag configuration */ 2783 err = nix_rx_vtag_cfg(rvu, nixlf, blkaddr, req); 2784 if (err) 2785 return NIX_AF_ERR_PARAM; 2786 } else { 2787 /* tx vtag configuration */ 2788 if ((req->tx.cfg_vtag0 || req->tx.cfg_vtag1) && 2789 (req->tx.free_vtag0 || req->tx.free_vtag1)) 2790 return NIX_AF_ERR_PARAM; 2791 2792 if (req->tx.cfg_vtag0 || req->tx.cfg_vtag1) 2793 return nix_tx_vtag_cfg(rvu, blkaddr, req, rsp); 2794 2795 if (req->tx.free_vtag0 || req->tx.free_vtag1) 2796 return nix_tx_vtag_decfg(rvu, blkaddr, req); 2797 } 2798 2799 return 0; 2800 } 2801 2802 static int nix_blk_setup_mce(struct rvu *rvu, struct nix_hw *nix_hw, 2803 int mce, u8 op, u16 pcifunc, int next, bool eol) 2804 { 2805 struct nix_aq_enq_req aq_req; 2806 int err; 2807 2808 aq_req.hdr.pcifunc = 0; 2809 aq_req.ctype = NIX_AQ_CTYPE_MCE; 2810 aq_req.op = op; 2811 aq_req.qidx = mce; 2812 2813 /* Use RSS with RSS index 0 */ 2814 aq_req.mce.op = 1; 2815 aq_req.mce.index = 0; 2816 aq_req.mce.eol = eol; 2817 aq_req.mce.pf_func = pcifunc; 2818 aq_req.mce.next = next; 2819 2820 /* All fields valid */ 2821 *(u64 *)(&aq_req.mce_mask) = ~0ULL; 2822 2823 err = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, &aq_req, NULL); 2824 if (err) { 2825 dev_err(rvu->dev, "Failed to setup Bcast MCE for PF%d:VF%d\n", 2826 rvu_get_pf(pcifunc), pcifunc & RVU_PFVF_FUNC_MASK); 2827 return err; 2828 } 2829 return 0; 2830 } 2831 2832 static int nix_update_mce_list_entry(struct nix_mce_list *mce_list, 2833 u16 pcifunc, bool add) 2834 { 2835 struct mce *mce, *tail = NULL; 2836 bool delete = false; 2837 2838 /* Scan through the current list */ 2839 hlist_for_each_entry(mce, &mce_list->head, node) { 2840 /* If already exists, then delete */ 2841 if (mce->pcifunc == pcifunc && !add) { 2842 delete = true; 2843 break; 2844 } else if (mce->pcifunc == pcifunc && add) { 2845 /* entry already exists */ 2846 return 0; 2847 } 2848 tail = mce; 2849 } 2850 2851 if (delete) { 2852 hlist_del(&mce->node); 2853 kfree(mce); 2854 mce_list->count--; 2855 return 0; 2856 } 2857 2858 if (!add) 2859 return 0; 2860 2861 /* Add a new one to the list, at the tail */ 2862 mce = kzalloc(sizeof(*mce), GFP_KERNEL); 2863 if (!mce) 2864 return -ENOMEM; 2865 mce->pcifunc = pcifunc; 2866 if (!tail) 2867 hlist_add_head(&mce->node, &mce_list->head); 2868 else 2869 hlist_add_behind(&mce->node, &tail->node); 2870 mce_list->count++; 2871 return 0; 2872 } 2873 2874 int nix_update_mce_list(struct rvu *rvu, u16 pcifunc, 2875 struct nix_mce_list *mce_list, 2876 int mce_idx, int mcam_index, bool add) 2877 { 2878 int err = 0, idx, next_idx, last_idx, blkaddr, npc_blkaddr; 2879 struct npc_mcam *mcam = &rvu->hw->mcam; 2880 struct nix_mcast *mcast; 2881 struct nix_hw *nix_hw; 2882 struct mce *mce; 2883 2884 if (!mce_list) 2885 return -EINVAL; 2886 2887 /* Get this PF/VF func's MCE index */ 2888 idx = mce_idx + (pcifunc & RVU_PFVF_FUNC_MASK); 2889 2890 if (idx > (mce_idx + mce_list->max)) { 2891 dev_err(rvu->dev, 2892 "%s: Idx %d > max MCE idx %d, for PF%d bcast list\n", 2893 __func__, idx, mce_list->max, 2894 pcifunc >> RVU_PFVF_PF_SHIFT); 2895 return -EINVAL; 2896 } 2897 2898 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 2899 if (err) 2900 return err; 2901 2902 mcast = &nix_hw->mcast; 2903 mutex_lock(&mcast->mce_lock); 2904 2905 err = nix_update_mce_list_entry(mce_list, pcifunc, add); 2906 if (err) 2907 goto end; 2908 2909 /* Disable MCAM entry in NPC */ 2910 if (!mce_list->count) { 2911 npc_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); 2912 npc_enable_mcam_entry(rvu, mcam, npc_blkaddr, mcam_index, false); 2913 goto end; 2914 } 2915 2916 /* Dump the updated list to HW */ 2917 idx = mce_idx; 2918 last_idx = idx + mce_list->count - 1; 2919 hlist_for_each_entry(mce, &mce_list->head, node) { 2920 if (idx > last_idx) 2921 break; 2922 2923 next_idx = idx + 1; 2924 /* EOL should be set in last MCE */ 2925 err = nix_blk_setup_mce(rvu, nix_hw, idx, NIX_AQ_INSTOP_WRITE, 2926 mce->pcifunc, next_idx, 2927 (next_idx > last_idx) ? true : false); 2928 if (err) 2929 goto end; 2930 idx++; 2931 } 2932 2933 end: 2934 mutex_unlock(&mcast->mce_lock); 2935 return err; 2936 } 2937 2938 void nix_get_mce_list(struct rvu *rvu, u16 pcifunc, int type, 2939 struct nix_mce_list **mce_list, int *mce_idx) 2940 { 2941 struct rvu_hwinfo *hw = rvu->hw; 2942 struct rvu_pfvf *pfvf; 2943 2944 if (!hw->cap.nix_rx_multicast || 2945 !is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc & ~RVU_PFVF_FUNC_MASK))) { 2946 *mce_list = NULL; 2947 *mce_idx = 0; 2948 return; 2949 } 2950 2951 /* Get this PF/VF func's MCE index */ 2952 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK); 2953 2954 if (type == NIXLF_BCAST_ENTRY) { 2955 *mce_list = &pfvf->bcast_mce_list; 2956 *mce_idx = pfvf->bcast_mce_idx; 2957 } else if (type == NIXLF_ALLMULTI_ENTRY) { 2958 *mce_list = &pfvf->mcast_mce_list; 2959 *mce_idx = pfvf->mcast_mce_idx; 2960 } else if (type == NIXLF_PROMISC_ENTRY) { 2961 *mce_list = &pfvf->promisc_mce_list; 2962 *mce_idx = pfvf->promisc_mce_idx; 2963 } else { 2964 *mce_list = NULL; 2965 *mce_idx = 0; 2966 } 2967 } 2968 2969 static int nix_update_mce_rule(struct rvu *rvu, u16 pcifunc, 2970 int type, bool add) 2971 { 2972 int err = 0, nixlf, blkaddr, mcam_index, mce_idx; 2973 struct npc_mcam *mcam = &rvu->hw->mcam; 2974 struct rvu_hwinfo *hw = rvu->hw; 2975 struct nix_mce_list *mce_list; 2976 int pf; 2977 2978 /* skip multicast pkt replication for AF's VFs & SDP links */ 2979 if (is_afvf(pcifunc) || is_sdp_pfvf(pcifunc)) 2980 return 0; 2981 2982 if (!hw->cap.nix_rx_multicast) 2983 return 0; 2984 2985 pf = rvu_get_pf(pcifunc); 2986 if (!is_pf_cgxmapped(rvu, pf)) 2987 return 0; 2988 2989 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 2990 if (blkaddr < 0) 2991 return -EINVAL; 2992 2993 nixlf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0); 2994 if (nixlf < 0) 2995 return -EINVAL; 2996 2997 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx); 2998 2999 mcam_index = npc_get_nixlf_mcam_index(mcam, 3000 pcifunc & ~RVU_PFVF_FUNC_MASK, 3001 nixlf, type); 3002 err = nix_update_mce_list(rvu, pcifunc, mce_list, 3003 mce_idx, mcam_index, add); 3004 return err; 3005 } 3006 3007 static int nix_setup_mce_tables(struct rvu *rvu, struct nix_hw *nix_hw) 3008 { 3009 struct nix_mcast *mcast = &nix_hw->mcast; 3010 int err, pf, numvfs, idx; 3011 struct rvu_pfvf *pfvf; 3012 u16 pcifunc; 3013 u64 cfg; 3014 3015 /* Skip PF0 (i.e AF) */ 3016 for (pf = 1; pf < (rvu->cgx_mapped_pfs + 1); pf++) { 3017 cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); 3018 /* If PF is not enabled, nothing to do */ 3019 if (!((cfg >> 20) & 0x01)) 3020 continue; 3021 /* Get numVFs attached to this PF */ 3022 numvfs = (cfg >> 12) & 0xFF; 3023 3024 pfvf = &rvu->pf[pf]; 3025 3026 /* This NIX0/1 block mapped to PF ? */ 3027 if (pfvf->nix_blkaddr != nix_hw->blkaddr) 3028 continue; 3029 3030 /* save start idx of broadcast mce list */ 3031 pfvf->bcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); 3032 nix_mce_list_init(&pfvf->bcast_mce_list, numvfs + 1); 3033 3034 /* save start idx of multicast mce list */ 3035 pfvf->mcast_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); 3036 nix_mce_list_init(&pfvf->mcast_mce_list, numvfs + 1); 3037 3038 /* save the start idx of promisc mce list */ 3039 pfvf->promisc_mce_idx = nix_alloc_mce_list(mcast, numvfs + 1); 3040 nix_mce_list_init(&pfvf->promisc_mce_list, numvfs + 1); 3041 3042 for (idx = 0; idx < (numvfs + 1); idx++) { 3043 /* idx-0 is for PF, followed by VFs */ 3044 pcifunc = (pf << RVU_PFVF_PF_SHIFT); 3045 pcifunc |= idx; 3046 /* Add dummy entries now, so that we don't have to check 3047 * for whether AQ_OP should be INIT/WRITE later on. 3048 * Will be updated when a NIXLF is attached/detached to 3049 * these PF/VFs. 3050 */ 3051 err = nix_blk_setup_mce(rvu, nix_hw, 3052 pfvf->bcast_mce_idx + idx, 3053 NIX_AQ_INSTOP_INIT, 3054 pcifunc, 0, true); 3055 if (err) 3056 return err; 3057 3058 /* add dummy entries to multicast mce list */ 3059 err = nix_blk_setup_mce(rvu, nix_hw, 3060 pfvf->mcast_mce_idx + idx, 3061 NIX_AQ_INSTOP_INIT, 3062 pcifunc, 0, true); 3063 if (err) 3064 return err; 3065 3066 /* add dummy entries to promisc mce list */ 3067 err = nix_blk_setup_mce(rvu, nix_hw, 3068 pfvf->promisc_mce_idx + idx, 3069 NIX_AQ_INSTOP_INIT, 3070 pcifunc, 0, true); 3071 if (err) 3072 return err; 3073 } 3074 } 3075 return 0; 3076 } 3077 3078 static int nix_setup_mcast(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3079 { 3080 struct nix_mcast *mcast = &nix_hw->mcast; 3081 struct rvu_hwinfo *hw = rvu->hw; 3082 int err, size; 3083 3084 size = (rvu_read64(rvu, blkaddr, NIX_AF_CONST3) >> 16) & 0x0F; 3085 size = (1ULL << size); 3086 3087 /* Alloc memory for multicast/mirror replication entries */ 3088 err = qmem_alloc(rvu->dev, &mcast->mce_ctx, 3089 (256UL << MC_TBL_SIZE), size); 3090 if (err) 3091 return -ENOMEM; 3092 3093 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BASE, 3094 (u64)mcast->mce_ctx->iova); 3095 3096 /* Set max list length equal to max no of VFs per PF + PF itself */ 3097 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_CFG, 3098 BIT_ULL(36) | (hw->max_vfs_per_pf << 4) | MC_TBL_SIZE); 3099 3100 /* Alloc memory for multicast replication buffers */ 3101 size = rvu_read64(rvu, blkaddr, NIX_AF_MC_MIRROR_CONST) & 0xFFFF; 3102 err = qmem_alloc(rvu->dev, &mcast->mcast_buf, 3103 (8UL << MC_BUF_CNT), size); 3104 if (err) 3105 return -ENOMEM; 3106 3107 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_BASE, 3108 (u64)mcast->mcast_buf->iova); 3109 3110 /* Alloc pkind for NIX internal RX multicast/mirror replay */ 3111 mcast->replay_pkind = rvu_alloc_rsrc(&hw->pkind.rsrc); 3112 3113 rvu_write64(rvu, blkaddr, NIX_AF_RX_MCAST_BUF_CFG, 3114 BIT_ULL(63) | (mcast->replay_pkind << 24) | 3115 BIT_ULL(20) | MC_BUF_CNT); 3116 3117 mutex_init(&mcast->mce_lock); 3118 3119 return nix_setup_mce_tables(rvu, nix_hw); 3120 } 3121 3122 static int nix_setup_txvlan(struct rvu *rvu, struct nix_hw *nix_hw) 3123 { 3124 struct nix_txvlan *vlan = &nix_hw->txvlan; 3125 int err; 3126 3127 /* Allocate resource bimap for tx vtag def registers*/ 3128 vlan->rsrc.max = NIX_TX_VTAG_DEF_MAX; 3129 err = rvu_alloc_bitmap(&vlan->rsrc); 3130 if (err) 3131 return -ENOMEM; 3132 3133 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */ 3134 vlan->entry2pfvf_map = devm_kcalloc(rvu->dev, vlan->rsrc.max, 3135 sizeof(u16), GFP_KERNEL); 3136 if (!vlan->entry2pfvf_map) 3137 goto free_mem; 3138 3139 mutex_init(&vlan->rsrc_lock); 3140 return 0; 3141 3142 free_mem: 3143 kfree(vlan->rsrc.bmap); 3144 return -ENOMEM; 3145 } 3146 3147 static int nix_setup_txschq(struct rvu *rvu, struct nix_hw *nix_hw, int blkaddr) 3148 { 3149 struct nix_txsch *txsch; 3150 int err, lvl, schq; 3151 u64 cfg, reg; 3152 3153 /* Get scheduler queue count of each type and alloc 3154 * bitmap for each for alloc/free/attach operations. 3155 */ 3156 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 3157 txsch = &nix_hw->txsch[lvl]; 3158 txsch->lvl = lvl; 3159 switch (lvl) { 3160 case NIX_TXSCH_LVL_SMQ: 3161 reg = NIX_AF_MDQ_CONST; 3162 break; 3163 case NIX_TXSCH_LVL_TL4: 3164 reg = NIX_AF_TL4_CONST; 3165 break; 3166 case NIX_TXSCH_LVL_TL3: 3167 reg = NIX_AF_TL3_CONST; 3168 break; 3169 case NIX_TXSCH_LVL_TL2: 3170 reg = NIX_AF_TL2_CONST; 3171 break; 3172 case NIX_TXSCH_LVL_TL1: 3173 reg = NIX_AF_TL1_CONST; 3174 break; 3175 } 3176 cfg = rvu_read64(rvu, blkaddr, reg); 3177 txsch->schq.max = cfg & 0xFFFF; 3178 err = rvu_alloc_bitmap(&txsch->schq); 3179 if (err) 3180 return err; 3181 3182 /* Allocate memory for scheduler queues to 3183 * PF/VF pcifunc mapping info. 3184 */ 3185 txsch->pfvf_map = devm_kcalloc(rvu->dev, txsch->schq.max, 3186 sizeof(u32), GFP_KERNEL); 3187 if (!txsch->pfvf_map) 3188 return -ENOMEM; 3189 for (schq = 0; schq < txsch->schq.max; schq++) 3190 txsch->pfvf_map[schq] = TXSCH_MAP(0, NIX_TXSCHQ_FREE); 3191 } 3192 3193 /* Setup a default value of 8192 as DWRR MTU */ 3194 if (rvu->hw->cap.nix_common_dwrr_mtu) { 3195 rvu_write64(rvu, blkaddr, NIX_AF_DWRR_RPM_MTU, 3196 convert_bytes_to_dwrr_mtu(8192)); 3197 rvu_write64(rvu, blkaddr, NIX_AF_DWRR_SDP_MTU, 3198 convert_bytes_to_dwrr_mtu(8192)); 3199 } 3200 3201 return 0; 3202 } 3203 3204 int rvu_nix_reserve_mark_format(struct rvu *rvu, struct nix_hw *nix_hw, 3205 int blkaddr, u32 cfg) 3206 { 3207 int fmt_idx; 3208 3209 for (fmt_idx = 0; fmt_idx < nix_hw->mark_format.in_use; fmt_idx++) { 3210 if (nix_hw->mark_format.cfg[fmt_idx] == cfg) 3211 return fmt_idx; 3212 } 3213 if (fmt_idx >= nix_hw->mark_format.total) 3214 return -ERANGE; 3215 3216 rvu_write64(rvu, blkaddr, NIX_AF_MARK_FORMATX_CTL(fmt_idx), cfg); 3217 nix_hw->mark_format.cfg[fmt_idx] = cfg; 3218 nix_hw->mark_format.in_use++; 3219 return fmt_idx; 3220 } 3221 3222 static int nix_af_mark_format_setup(struct rvu *rvu, struct nix_hw *nix_hw, 3223 int blkaddr) 3224 { 3225 u64 cfgs[] = { 3226 [NIX_MARK_CFG_IP_DSCP_RED] = 0x10003, 3227 [NIX_MARK_CFG_IP_DSCP_YELLOW] = 0x11200, 3228 [NIX_MARK_CFG_IP_DSCP_YELLOW_RED] = 0x11203, 3229 [NIX_MARK_CFG_IP_ECN_RED] = 0x6000c, 3230 [NIX_MARK_CFG_IP_ECN_YELLOW] = 0x60c00, 3231 [NIX_MARK_CFG_IP_ECN_YELLOW_RED] = 0x60c0c, 3232 [NIX_MARK_CFG_VLAN_DEI_RED] = 0x30008, 3233 [NIX_MARK_CFG_VLAN_DEI_YELLOW] = 0x30800, 3234 [NIX_MARK_CFG_VLAN_DEI_YELLOW_RED] = 0x30808, 3235 }; 3236 int i, rc; 3237 u64 total; 3238 3239 total = (rvu_read64(rvu, blkaddr, NIX_AF_PSE_CONST) & 0xFF00) >> 8; 3240 nix_hw->mark_format.total = (u8)total; 3241 nix_hw->mark_format.cfg = devm_kcalloc(rvu->dev, total, sizeof(u32), 3242 GFP_KERNEL); 3243 if (!nix_hw->mark_format.cfg) 3244 return -ENOMEM; 3245 for (i = 0; i < NIX_MARK_CFG_MAX; i++) { 3246 rc = rvu_nix_reserve_mark_format(rvu, nix_hw, blkaddr, cfgs[i]); 3247 if (rc < 0) 3248 dev_err(rvu->dev, "Err %d in setup mark format %d\n", 3249 i, rc); 3250 } 3251 3252 return 0; 3253 } 3254 3255 static void rvu_get_lbk_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3256 { 3257 /* CN10K supports LBK FIFO size 72 KB */ 3258 if (rvu->hw->lbk_bufsize == 0x12000) 3259 *max_mtu = CN10K_LBK_LINK_MAX_FRS; 3260 else 3261 *max_mtu = NIC_HW_MAX_FRS; 3262 } 3263 3264 static void rvu_get_lmac_link_max_frs(struct rvu *rvu, u16 *max_mtu) 3265 { 3266 int fifo_size = rvu_cgx_get_fifolen(rvu); 3267 3268 /* RPM supports FIFO len 128 KB and RPM2 supports double the 3269 * FIFO len to accommodate 8 LMACS 3270 */ 3271 if (fifo_size == 0x20000 || fifo_size == 0x40000) 3272 *max_mtu = CN10K_LMAC_LINK_MAX_FRS; 3273 else 3274 *max_mtu = NIC_HW_MAX_FRS; 3275 } 3276 3277 int rvu_mbox_handler_nix_get_hw_info(struct rvu *rvu, struct msg_req *req, 3278 struct nix_hw_info *rsp) 3279 { 3280 u16 pcifunc = req->hdr.pcifunc; 3281 u64 dwrr_mtu; 3282 int blkaddr; 3283 3284 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 3285 if (blkaddr < 0) 3286 return NIX_AF_ERR_AF_LF_INVALID; 3287 3288 if (is_afvf(pcifunc)) 3289 rvu_get_lbk_link_max_frs(rvu, &rsp->max_mtu); 3290 else 3291 rvu_get_lmac_link_max_frs(rvu, &rsp->max_mtu); 3292 3293 rsp->min_mtu = NIC_HW_MIN_FRS; 3294 3295 if (!rvu->hw->cap.nix_common_dwrr_mtu) { 3296 /* Return '1' on OTx2 */ 3297 rsp->rpm_dwrr_mtu = 1; 3298 rsp->sdp_dwrr_mtu = 1; 3299 return 0; 3300 } 3301 3302 dwrr_mtu = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_DWRR_RPM_MTU); 3303 rsp->rpm_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3304 3305 dwrr_mtu = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_DWRR_SDP_MTU); 3306 rsp->sdp_dwrr_mtu = convert_dwrr_mtu_to_bytes(dwrr_mtu); 3307 3308 return 0; 3309 } 3310 3311 int rvu_mbox_handler_nix_stats_rst(struct rvu *rvu, struct msg_req *req, 3312 struct msg_rsp *rsp) 3313 { 3314 u16 pcifunc = req->hdr.pcifunc; 3315 int i, nixlf, blkaddr, err; 3316 u64 stats; 3317 3318 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3319 if (err) 3320 return err; 3321 3322 /* Get stats count supported by HW */ 3323 stats = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 3324 3325 /* Reset tx stats */ 3326 for (i = 0; i < ((stats >> 24) & 0xFF); i++) 3327 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_STATX(nixlf, i), 0); 3328 3329 /* Reset rx stats */ 3330 for (i = 0; i < ((stats >> 32) & 0xFF); i++) 3331 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_STATX(nixlf, i), 0); 3332 3333 return 0; 3334 } 3335 3336 /* Returns the ALG index to be set into NPC_RX_ACTION */ 3337 static int get_flowkey_alg_idx(struct nix_hw *nix_hw, u32 flow_cfg) 3338 { 3339 int i; 3340 3341 /* Scan over exiting algo entries to find a match */ 3342 for (i = 0; i < nix_hw->flowkey.in_use; i++) 3343 if (nix_hw->flowkey.flowkey[i] == flow_cfg) 3344 return i; 3345 3346 return -ERANGE; 3347 } 3348 3349 static int set_flowkey_fields(struct nix_rx_flowkey_alg *alg, u32 flow_cfg) 3350 { 3351 int idx, nr_field, key_off, field_marker, keyoff_marker; 3352 int max_key_off, max_bit_pos, group_member; 3353 struct nix_rx_flowkey_alg *field; 3354 struct nix_rx_flowkey_alg tmp; 3355 u32 key_type, valid_key; 3356 int l4_key_offset = 0; 3357 3358 if (!alg) 3359 return -EINVAL; 3360 3361 #define FIELDS_PER_ALG 5 3362 #define MAX_KEY_OFF 40 3363 /* Clear all fields */ 3364 memset(alg, 0, sizeof(uint64_t) * FIELDS_PER_ALG); 3365 3366 /* Each of the 32 possible flow key algorithm definitions should 3367 * fall into above incremental config (except ALG0). Otherwise a 3368 * single NPC MCAM entry is not sufficient for supporting RSS. 3369 * 3370 * If a different definition or combination needed then NPC MCAM 3371 * has to be programmed to filter such pkts and it's action should 3372 * point to this definition to calculate flowtag or hash. 3373 * 3374 * The `for loop` goes over _all_ protocol field and the following 3375 * variables depicts the state machine forward progress logic. 3376 * 3377 * keyoff_marker - Enabled when hash byte length needs to be accounted 3378 * in field->key_offset update. 3379 * field_marker - Enabled when a new field needs to be selected. 3380 * group_member - Enabled when protocol is part of a group. 3381 */ 3382 3383 keyoff_marker = 0; max_key_off = 0; group_member = 0; 3384 nr_field = 0; key_off = 0; field_marker = 1; 3385 field = &tmp; max_bit_pos = fls(flow_cfg); 3386 for (idx = 0; 3387 idx < max_bit_pos && nr_field < FIELDS_PER_ALG && 3388 key_off < MAX_KEY_OFF; idx++) { 3389 key_type = BIT(idx); 3390 valid_key = flow_cfg & key_type; 3391 /* Found a field marker, reset the field values */ 3392 if (field_marker) 3393 memset(&tmp, 0, sizeof(tmp)); 3394 3395 field_marker = true; 3396 keyoff_marker = true; 3397 switch (key_type) { 3398 case NIX_FLOW_KEY_TYPE_PORT: 3399 field->sel_chan = true; 3400 /* This should be set to 1, when SEL_CHAN is set */ 3401 field->bytesm1 = 1; 3402 break; 3403 case NIX_FLOW_KEY_TYPE_IPV4_PROTO: 3404 field->lid = NPC_LID_LC; 3405 field->hdr_offset = 9; /* offset */ 3406 field->bytesm1 = 0; /* 1 byte */ 3407 field->ltype_match = NPC_LT_LC_IP; 3408 field->ltype_mask = 0xF; 3409 break; 3410 case NIX_FLOW_KEY_TYPE_IPV4: 3411 case NIX_FLOW_KEY_TYPE_INNR_IPV4: 3412 field->lid = NPC_LID_LC; 3413 field->ltype_match = NPC_LT_LC_IP; 3414 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV4) { 3415 field->lid = NPC_LID_LG; 3416 field->ltype_match = NPC_LT_LG_TU_IP; 3417 } 3418 field->hdr_offset = 12; /* SIP offset */ 3419 field->bytesm1 = 7; /* SIP + DIP, 8 bytes */ 3420 field->ltype_mask = 0xF; /* Match only IPv4 */ 3421 keyoff_marker = false; 3422 break; 3423 case NIX_FLOW_KEY_TYPE_IPV6: 3424 case NIX_FLOW_KEY_TYPE_INNR_IPV6: 3425 field->lid = NPC_LID_LC; 3426 field->ltype_match = NPC_LT_LC_IP6; 3427 if (key_type == NIX_FLOW_KEY_TYPE_INNR_IPV6) { 3428 field->lid = NPC_LID_LG; 3429 field->ltype_match = NPC_LT_LG_TU_IP6; 3430 } 3431 field->hdr_offset = 8; /* SIP offset */ 3432 field->bytesm1 = 31; /* SIP + DIP, 32 bytes */ 3433 field->ltype_mask = 0xF; /* Match only IPv6 */ 3434 break; 3435 case NIX_FLOW_KEY_TYPE_TCP: 3436 case NIX_FLOW_KEY_TYPE_UDP: 3437 case NIX_FLOW_KEY_TYPE_SCTP: 3438 case NIX_FLOW_KEY_TYPE_INNR_TCP: 3439 case NIX_FLOW_KEY_TYPE_INNR_UDP: 3440 case NIX_FLOW_KEY_TYPE_INNR_SCTP: 3441 field->lid = NPC_LID_LD; 3442 if (key_type == NIX_FLOW_KEY_TYPE_INNR_TCP || 3443 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP || 3444 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) 3445 field->lid = NPC_LID_LH; 3446 field->bytesm1 = 3; /* Sport + Dport, 4 bytes */ 3447 3448 /* Enum values for NPC_LID_LD and NPC_LID_LG are same, 3449 * so no need to change the ltype_match, just change 3450 * the lid for inner protocols 3451 */ 3452 BUILD_BUG_ON((int)NPC_LT_LD_TCP != 3453 (int)NPC_LT_LH_TU_TCP); 3454 BUILD_BUG_ON((int)NPC_LT_LD_UDP != 3455 (int)NPC_LT_LH_TU_UDP); 3456 BUILD_BUG_ON((int)NPC_LT_LD_SCTP != 3457 (int)NPC_LT_LH_TU_SCTP); 3458 3459 if ((key_type == NIX_FLOW_KEY_TYPE_TCP || 3460 key_type == NIX_FLOW_KEY_TYPE_INNR_TCP) && 3461 valid_key) { 3462 field->ltype_match |= NPC_LT_LD_TCP; 3463 group_member = true; 3464 } else if ((key_type == NIX_FLOW_KEY_TYPE_UDP || 3465 key_type == NIX_FLOW_KEY_TYPE_INNR_UDP) && 3466 valid_key) { 3467 field->ltype_match |= NPC_LT_LD_UDP; 3468 group_member = true; 3469 } else if ((key_type == NIX_FLOW_KEY_TYPE_SCTP || 3470 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) && 3471 valid_key) { 3472 field->ltype_match |= NPC_LT_LD_SCTP; 3473 group_member = true; 3474 } 3475 field->ltype_mask = ~field->ltype_match; 3476 if (key_type == NIX_FLOW_KEY_TYPE_SCTP || 3477 key_type == NIX_FLOW_KEY_TYPE_INNR_SCTP) { 3478 /* Handle the case where any of the group item 3479 * is enabled in the group but not the final one 3480 */ 3481 if (group_member) { 3482 valid_key = true; 3483 group_member = false; 3484 } 3485 } else { 3486 field_marker = false; 3487 keyoff_marker = false; 3488 } 3489 3490 /* TCP/UDP/SCTP and ESP/AH falls at same offset so 3491 * remember the TCP key offset of 40 byte hash key. 3492 */ 3493 if (key_type == NIX_FLOW_KEY_TYPE_TCP) 3494 l4_key_offset = key_off; 3495 break; 3496 case NIX_FLOW_KEY_TYPE_NVGRE: 3497 field->lid = NPC_LID_LD; 3498 field->hdr_offset = 4; /* VSID offset */ 3499 field->bytesm1 = 2; 3500 field->ltype_match = NPC_LT_LD_NVGRE; 3501 field->ltype_mask = 0xF; 3502 break; 3503 case NIX_FLOW_KEY_TYPE_VXLAN: 3504 case NIX_FLOW_KEY_TYPE_GENEVE: 3505 field->lid = NPC_LID_LE; 3506 field->bytesm1 = 2; 3507 field->hdr_offset = 4; 3508 field->ltype_mask = 0xF; 3509 field_marker = false; 3510 keyoff_marker = false; 3511 3512 if (key_type == NIX_FLOW_KEY_TYPE_VXLAN && valid_key) { 3513 field->ltype_match |= NPC_LT_LE_VXLAN; 3514 group_member = true; 3515 } 3516 3517 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE && valid_key) { 3518 field->ltype_match |= NPC_LT_LE_GENEVE; 3519 group_member = true; 3520 } 3521 3522 if (key_type == NIX_FLOW_KEY_TYPE_GENEVE) { 3523 if (group_member) { 3524 field->ltype_mask = ~field->ltype_match; 3525 field_marker = true; 3526 keyoff_marker = true; 3527 valid_key = true; 3528 group_member = false; 3529 } 3530 } 3531 break; 3532 case NIX_FLOW_KEY_TYPE_ETH_DMAC: 3533 case NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC: 3534 field->lid = NPC_LID_LA; 3535 field->ltype_match = NPC_LT_LA_ETHER; 3536 if (key_type == NIX_FLOW_KEY_TYPE_INNR_ETH_DMAC) { 3537 field->lid = NPC_LID_LF; 3538 field->ltype_match = NPC_LT_LF_TU_ETHER; 3539 } 3540 field->hdr_offset = 0; 3541 field->bytesm1 = 5; /* DMAC 6 Byte */ 3542 field->ltype_mask = 0xF; 3543 break; 3544 case NIX_FLOW_KEY_TYPE_IPV6_EXT: 3545 field->lid = NPC_LID_LC; 3546 field->hdr_offset = 40; /* IPV6 hdr */ 3547 field->bytesm1 = 0; /* 1 Byte ext hdr*/ 3548 field->ltype_match = NPC_LT_LC_IP6_EXT; 3549 field->ltype_mask = 0xF; 3550 break; 3551 case NIX_FLOW_KEY_TYPE_GTPU: 3552 field->lid = NPC_LID_LE; 3553 field->hdr_offset = 4; 3554 field->bytesm1 = 3; /* 4 bytes TID*/ 3555 field->ltype_match = NPC_LT_LE_GTPU; 3556 field->ltype_mask = 0xF; 3557 break; 3558 case NIX_FLOW_KEY_TYPE_VLAN: 3559 field->lid = NPC_LID_LB; 3560 field->hdr_offset = 2; /* Skip TPID (2-bytes) */ 3561 field->bytesm1 = 1; /* 2 Bytes (Actually 12 bits) */ 3562 field->ltype_match = NPC_LT_LB_CTAG; 3563 field->ltype_mask = 0xF; 3564 field->fn_mask = 1; /* Mask out the first nibble */ 3565 break; 3566 case NIX_FLOW_KEY_TYPE_AH: 3567 case NIX_FLOW_KEY_TYPE_ESP: 3568 field->hdr_offset = 0; 3569 field->bytesm1 = 7; /* SPI + sequence number */ 3570 field->ltype_mask = 0xF; 3571 field->lid = NPC_LID_LE; 3572 field->ltype_match = NPC_LT_LE_ESP; 3573 if (key_type == NIX_FLOW_KEY_TYPE_AH) { 3574 field->lid = NPC_LID_LD; 3575 field->ltype_match = NPC_LT_LD_AH; 3576 field->hdr_offset = 4; 3577 keyoff_marker = false; 3578 } 3579 break; 3580 } 3581 field->ena = 1; 3582 3583 /* Found a valid flow key type */ 3584 if (valid_key) { 3585 /* Use the key offset of TCP/UDP/SCTP fields 3586 * for ESP/AH fields. 3587 */ 3588 if (key_type == NIX_FLOW_KEY_TYPE_ESP || 3589 key_type == NIX_FLOW_KEY_TYPE_AH) 3590 key_off = l4_key_offset; 3591 field->key_offset = key_off; 3592 memcpy(&alg[nr_field], field, sizeof(*field)); 3593 max_key_off = max(max_key_off, field->bytesm1 + 1); 3594 3595 /* Found a field marker, get the next field */ 3596 if (field_marker) 3597 nr_field++; 3598 } 3599 3600 /* Found a keyoff marker, update the new key_off */ 3601 if (keyoff_marker) { 3602 key_off += max_key_off; 3603 max_key_off = 0; 3604 } 3605 } 3606 /* Processed all the flow key types */ 3607 if (idx == max_bit_pos && key_off <= MAX_KEY_OFF) 3608 return 0; 3609 else 3610 return NIX_AF_ERR_RSS_NOSPC_FIELD; 3611 } 3612 3613 static int reserve_flowkey_alg_idx(struct rvu *rvu, int blkaddr, u32 flow_cfg) 3614 { 3615 u64 field[FIELDS_PER_ALG]; 3616 struct nix_hw *hw; 3617 int fid, rc; 3618 3619 hw = get_nix_hw(rvu->hw, blkaddr); 3620 if (!hw) 3621 return NIX_AF_ERR_INVALID_NIXBLK; 3622 3623 /* No room to add new flow hash algoritham */ 3624 if (hw->flowkey.in_use >= NIX_FLOW_KEY_ALG_MAX) 3625 return NIX_AF_ERR_RSS_NOSPC_ALGO; 3626 3627 /* Generate algo fields for the given flow_cfg */ 3628 rc = set_flowkey_fields((struct nix_rx_flowkey_alg *)field, flow_cfg); 3629 if (rc) 3630 return rc; 3631 3632 /* Update ALGX_FIELDX register with generated fields */ 3633 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 3634 rvu_write64(rvu, blkaddr, 3635 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(hw->flowkey.in_use, 3636 fid), field[fid]); 3637 3638 /* Store the flow_cfg for futher lookup */ 3639 rc = hw->flowkey.in_use; 3640 hw->flowkey.flowkey[rc] = flow_cfg; 3641 hw->flowkey.in_use++; 3642 3643 return rc; 3644 } 3645 3646 int rvu_mbox_handler_nix_rss_flowkey_cfg(struct rvu *rvu, 3647 struct nix_rss_flowkey_cfg *req, 3648 struct nix_rss_flowkey_cfg_rsp *rsp) 3649 { 3650 u16 pcifunc = req->hdr.pcifunc; 3651 int alg_idx, nixlf, blkaddr; 3652 struct nix_hw *nix_hw; 3653 int err; 3654 3655 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3656 if (err) 3657 return err; 3658 3659 nix_hw = get_nix_hw(rvu->hw, blkaddr); 3660 if (!nix_hw) 3661 return NIX_AF_ERR_INVALID_NIXBLK; 3662 3663 alg_idx = get_flowkey_alg_idx(nix_hw, req->flowkey_cfg); 3664 /* Failed to get algo index from the exiting list, reserve new */ 3665 if (alg_idx < 0) { 3666 alg_idx = reserve_flowkey_alg_idx(rvu, blkaddr, 3667 req->flowkey_cfg); 3668 if (alg_idx < 0) 3669 return alg_idx; 3670 } 3671 rsp->alg_idx = alg_idx; 3672 rvu_npc_update_flowkey_alg_idx(rvu, pcifunc, nixlf, req->group, 3673 alg_idx, req->mcam_index); 3674 return 0; 3675 } 3676 3677 static int nix_rx_flowkey_alg_cfg(struct rvu *rvu, int blkaddr) 3678 { 3679 u32 flowkey_cfg, minkey_cfg; 3680 int alg, fid, rc; 3681 3682 /* Disable all flow key algx fieldx */ 3683 for (alg = 0; alg < NIX_FLOW_KEY_ALG_MAX; alg++) { 3684 for (fid = 0; fid < FIELDS_PER_ALG; fid++) 3685 rvu_write64(rvu, blkaddr, 3686 NIX_AF_RX_FLOW_KEY_ALGX_FIELDX(alg, fid), 3687 0); 3688 } 3689 3690 /* IPv4/IPv6 SIP/DIPs */ 3691 flowkey_cfg = NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6; 3692 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3693 if (rc < 0) 3694 return rc; 3695 3696 /* TCPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 3697 minkey_cfg = flowkey_cfg; 3698 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP; 3699 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3700 if (rc < 0) 3701 return rc; 3702 3703 /* UDPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 3704 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP; 3705 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3706 if (rc < 0) 3707 return rc; 3708 3709 /* SCTPv4/v6 4-tuple, SIP, DIP, Sport, Dport */ 3710 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_SCTP; 3711 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3712 if (rc < 0) 3713 return rc; 3714 3715 /* TCP/UDP v4/v6 4-tuple, rest IP pkts 2-tuple */ 3716 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 3717 NIX_FLOW_KEY_TYPE_UDP; 3718 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3719 if (rc < 0) 3720 return rc; 3721 3722 /* TCP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 3723 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 3724 NIX_FLOW_KEY_TYPE_SCTP; 3725 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3726 if (rc < 0) 3727 return rc; 3728 3729 /* UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 3730 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_UDP | 3731 NIX_FLOW_KEY_TYPE_SCTP; 3732 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3733 if (rc < 0) 3734 return rc; 3735 3736 /* TCP/UDP/SCTP v4/v6 4-tuple, rest IP pkts 2-tuple */ 3737 flowkey_cfg = minkey_cfg | NIX_FLOW_KEY_TYPE_TCP | 3738 NIX_FLOW_KEY_TYPE_UDP | NIX_FLOW_KEY_TYPE_SCTP; 3739 rc = reserve_flowkey_alg_idx(rvu, blkaddr, flowkey_cfg); 3740 if (rc < 0) 3741 return rc; 3742 3743 return 0; 3744 } 3745 3746 int rvu_mbox_handler_nix_set_mac_addr(struct rvu *rvu, 3747 struct nix_set_mac_addr *req, 3748 struct msg_rsp *rsp) 3749 { 3750 bool from_vf = req->hdr.pcifunc & RVU_PFVF_FUNC_MASK; 3751 u16 pcifunc = req->hdr.pcifunc; 3752 int blkaddr, nixlf, err; 3753 struct rvu_pfvf *pfvf; 3754 3755 err = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr); 3756 if (err) 3757 return err; 3758 3759 pfvf = rvu_get_pfvf(rvu, pcifunc); 3760 3761 /* untrusted VF can't overwrite admin(PF) changes */ 3762 if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 3763 (from_vf && test_bit(PF_SET_VF_MAC, &pfvf->flags))) { 3764 dev_warn(rvu->dev, 3765 "MAC address set by admin(PF) cannot be overwritten by untrusted VF"); 3766 return -EPERM; 3767 } 3768 3769 ether_addr_copy(pfvf->mac_addr, req->mac_addr); 3770 3771 rvu_npc_install_ucast_entry(rvu, pcifunc, nixlf, 3772 pfvf->rx_chan_base, req->mac_addr); 3773 3774 if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && from_vf) 3775 ether_addr_copy(pfvf->default_mac, req->mac_addr); 3776 3777 rvu_switch_update_rules(rvu, pcifunc); 3778 3779 return 0; 3780 } 3781 3782 int rvu_mbox_handler_nix_get_mac_addr(struct rvu *rvu, 3783 struct msg_req *req, 3784 struct nix_get_mac_addr_rsp *rsp) 3785 { 3786 u16 pcifunc = req->hdr.pcifunc; 3787 struct rvu_pfvf *pfvf; 3788 3789 if (!is_nixlf_attached(rvu, pcifunc)) 3790 return NIX_AF_ERR_AF_LF_INVALID; 3791 3792 pfvf = rvu_get_pfvf(rvu, pcifunc); 3793 3794 ether_addr_copy(rsp->mac_addr, pfvf->mac_addr); 3795 3796 return 0; 3797 } 3798 3799 int rvu_mbox_handler_nix_set_rx_mode(struct rvu *rvu, struct nix_rx_mode *req, 3800 struct msg_rsp *rsp) 3801 { 3802 bool allmulti, promisc, nix_rx_multicast; 3803 u16 pcifunc = req->hdr.pcifunc; 3804 struct rvu_pfvf *pfvf; 3805 int nixlf, err; 3806 3807 pfvf = rvu_get_pfvf(rvu, pcifunc); 3808 promisc = req->mode & NIX_RX_MODE_PROMISC ? true : false; 3809 allmulti = req->mode & NIX_RX_MODE_ALLMULTI ? true : false; 3810 pfvf->use_mce_list = req->mode & NIX_RX_MODE_USE_MCE ? true : false; 3811 3812 nix_rx_multicast = rvu->hw->cap.nix_rx_multicast & pfvf->use_mce_list; 3813 3814 if (is_vf(pcifunc) && !nix_rx_multicast && 3815 (promisc || allmulti)) { 3816 dev_warn_ratelimited(rvu->dev, 3817 "VF promisc/multicast not supported\n"); 3818 return 0; 3819 } 3820 3821 /* untrusted VF can't configure promisc/allmulti */ 3822 if (is_vf(pcifunc) && !test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) && 3823 (promisc || allmulti)) 3824 return 0; 3825 3826 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 3827 if (err) 3828 return err; 3829 3830 if (nix_rx_multicast) { 3831 /* add/del this PF_FUNC to/from mcast pkt replication list */ 3832 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_ALLMULTI_ENTRY, 3833 allmulti); 3834 if (err) { 3835 dev_err(rvu->dev, 3836 "Failed to update pcifunc 0x%x to multicast list\n", 3837 pcifunc); 3838 return err; 3839 } 3840 3841 /* add/del this PF_FUNC to/from promisc pkt replication list */ 3842 err = nix_update_mce_rule(rvu, pcifunc, NIXLF_PROMISC_ENTRY, 3843 promisc); 3844 if (err) { 3845 dev_err(rvu->dev, 3846 "Failed to update pcifunc 0x%x to promisc list\n", 3847 pcifunc); 3848 return err; 3849 } 3850 } 3851 3852 /* install/uninstall allmulti entry */ 3853 if (allmulti) { 3854 rvu_npc_install_allmulti_entry(rvu, pcifunc, nixlf, 3855 pfvf->rx_chan_base); 3856 } else { 3857 if (!nix_rx_multicast) 3858 rvu_npc_enable_allmulti_entry(rvu, pcifunc, nixlf, false); 3859 } 3860 3861 /* install/uninstall promisc entry */ 3862 if (promisc) { 3863 rvu_npc_install_promisc_entry(rvu, pcifunc, nixlf, 3864 pfvf->rx_chan_base, 3865 pfvf->rx_chan_cnt); 3866 3867 if (rvu_npc_exact_has_match_table(rvu)) 3868 rvu_npc_exact_promisc_enable(rvu, pcifunc); 3869 } else { 3870 if (!nix_rx_multicast) 3871 rvu_npc_enable_promisc_entry(rvu, pcifunc, nixlf, false); 3872 3873 if (rvu_npc_exact_has_match_table(rvu)) 3874 rvu_npc_exact_promisc_disable(rvu, pcifunc); 3875 } 3876 3877 return 0; 3878 } 3879 3880 static void nix_find_link_frs(struct rvu *rvu, 3881 struct nix_frs_cfg *req, u16 pcifunc) 3882 { 3883 int pf = rvu_get_pf(pcifunc); 3884 struct rvu_pfvf *pfvf; 3885 int maxlen, minlen; 3886 int numvfs, hwvf; 3887 int vf; 3888 3889 /* Update with requester's min/max lengths */ 3890 pfvf = rvu_get_pfvf(rvu, pcifunc); 3891 pfvf->maxlen = req->maxlen; 3892 if (req->update_minlen) 3893 pfvf->minlen = req->minlen; 3894 3895 maxlen = req->maxlen; 3896 minlen = req->update_minlen ? req->minlen : 0; 3897 3898 /* Get this PF's numVFs and starting hwvf */ 3899 rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf); 3900 3901 /* For each VF, compare requested max/minlen */ 3902 for (vf = 0; vf < numvfs; vf++) { 3903 pfvf = &rvu->hwvf[hwvf + vf]; 3904 if (pfvf->maxlen > maxlen) 3905 maxlen = pfvf->maxlen; 3906 if (req->update_minlen && 3907 pfvf->minlen && pfvf->minlen < minlen) 3908 minlen = pfvf->minlen; 3909 } 3910 3911 /* Compare requested max/minlen with PF's max/minlen */ 3912 pfvf = &rvu->pf[pf]; 3913 if (pfvf->maxlen > maxlen) 3914 maxlen = pfvf->maxlen; 3915 if (req->update_minlen && 3916 pfvf->minlen && pfvf->minlen < minlen) 3917 minlen = pfvf->minlen; 3918 3919 /* Update the request with max/min PF's and it's VF's max/min */ 3920 req->maxlen = maxlen; 3921 if (req->update_minlen) 3922 req->minlen = minlen; 3923 } 3924 3925 static int 3926 nix_config_link_credits(struct rvu *rvu, int blkaddr, int link, 3927 u16 pcifunc, u64 tx_credits) 3928 { 3929 struct rvu_hwinfo *hw = rvu->hw; 3930 int pf = rvu_get_pf(pcifunc); 3931 u8 cgx_id = 0, lmac_id = 0; 3932 unsigned long poll_tmo; 3933 bool restore_tx_en = 0; 3934 struct nix_hw *nix_hw; 3935 u64 cfg, sw_xoff = 0; 3936 u32 schq = 0; 3937 u32 credits; 3938 int rc; 3939 3940 nix_hw = get_nix_hw(rvu->hw, blkaddr); 3941 if (!nix_hw) 3942 return NIX_AF_ERR_INVALID_NIXBLK; 3943 3944 if (tx_credits == nix_hw->tx_credits[link]) 3945 return 0; 3946 3947 /* Enable cgx tx if disabled for credits to be back */ 3948 if (is_pf_cgxmapped(rvu, pf)) { 3949 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 3950 restore_tx_en = !rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), 3951 lmac_id, true); 3952 } 3953 3954 mutex_lock(&rvu->rsrc_lock); 3955 /* Disable new traffic to link */ 3956 if (hw->cap.nix_shaping) { 3957 schq = nix_get_tx_link(rvu, pcifunc); 3958 sw_xoff = rvu_read64(rvu, blkaddr, NIX_AF_TL1X_SW_XOFF(schq)); 3959 rvu_write64(rvu, blkaddr, 3960 NIX_AF_TL1X_SW_XOFF(schq), BIT_ULL(0)); 3961 } 3962 3963 rc = NIX_AF_ERR_LINK_CREDITS; 3964 poll_tmo = jiffies + usecs_to_jiffies(200000); 3965 /* Wait for credits to return */ 3966 do { 3967 if (time_after(jiffies, poll_tmo)) 3968 goto exit; 3969 usleep_range(100, 200); 3970 3971 cfg = rvu_read64(rvu, blkaddr, 3972 NIX_AF_TX_LINKX_NORM_CREDIT(link)); 3973 credits = (cfg >> 12) & 0xFFFFFULL; 3974 } while (credits != nix_hw->tx_credits[link]); 3975 3976 cfg &= ~(0xFFFFFULL << 12); 3977 cfg |= (tx_credits << 12); 3978 rvu_write64(rvu, blkaddr, NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); 3979 rc = 0; 3980 3981 nix_hw->tx_credits[link] = tx_credits; 3982 3983 exit: 3984 /* Enable traffic back */ 3985 if (hw->cap.nix_shaping && !sw_xoff) 3986 rvu_write64(rvu, blkaddr, NIX_AF_TL1X_SW_XOFF(schq), 0); 3987 3988 /* Restore state of cgx tx */ 3989 if (restore_tx_en) 3990 rvu_cgx_config_tx(rvu_cgx_pdata(cgx_id, rvu), lmac_id, false); 3991 3992 mutex_unlock(&rvu->rsrc_lock); 3993 return rc; 3994 } 3995 3996 int rvu_mbox_handler_nix_set_hw_frs(struct rvu *rvu, struct nix_frs_cfg *req, 3997 struct msg_rsp *rsp) 3998 { 3999 struct rvu_hwinfo *hw = rvu->hw; 4000 u16 pcifunc = req->hdr.pcifunc; 4001 int pf = rvu_get_pf(pcifunc); 4002 int blkaddr, schq, link = -1; 4003 struct nix_txsch *txsch; 4004 u64 cfg, lmac_fifo_len; 4005 struct nix_hw *nix_hw; 4006 struct rvu_pfvf *pfvf; 4007 u8 cgx = 0, lmac = 0; 4008 u16 max_mtu; 4009 4010 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 4011 if (blkaddr < 0) 4012 return NIX_AF_ERR_AF_LF_INVALID; 4013 4014 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4015 if (!nix_hw) 4016 return NIX_AF_ERR_INVALID_NIXBLK; 4017 4018 if (is_afvf(pcifunc)) 4019 rvu_get_lbk_link_max_frs(rvu, &max_mtu); 4020 else 4021 rvu_get_lmac_link_max_frs(rvu, &max_mtu); 4022 4023 if (!req->sdp_link && req->maxlen > max_mtu) 4024 return NIX_AF_ERR_FRS_INVALID; 4025 4026 if (req->update_minlen && req->minlen < NIC_HW_MIN_FRS) 4027 return NIX_AF_ERR_FRS_INVALID; 4028 4029 /* Check if requester wants to update SMQ's */ 4030 if (!req->update_smq) 4031 goto rx_frscfg; 4032 4033 /* Update min/maxlen in each of the SMQ attached to this PF/VF */ 4034 txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ]; 4035 mutex_lock(&rvu->rsrc_lock); 4036 for (schq = 0; schq < txsch->schq.max; schq++) { 4037 if (TXSCH_MAP_FUNC(txsch->pfvf_map[schq]) != pcifunc) 4038 continue; 4039 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq)); 4040 cfg = (cfg & ~(0xFFFFULL << 8)) | ((u64)req->maxlen << 8); 4041 if (req->update_minlen) 4042 cfg = (cfg & ~0x7FULL) | ((u64)req->minlen & 0x7F); 4043 rvu_write64(rvu, blkaddr, NIX_AF_SMQX_CFG(schq), cfg); 4044 } 4045 mutex_unlock(&rvu->rsrc_lock); 4046 4047 rx_frscfg: 4048 /* Check if config is for SDP link */ 4049 if (req->sdp_link) { 4050 if (!hw->sdp_links) 4051 return NIX_AF_ERR_RX_LINK_INVALID; 4052 link = hw->cgx_links + hw->lbk_links; 4053 goto linkcfg; 4054 } 4055 4056 /* Check if the request is from CGX mapped RVU PF */ 4057 if (is_pf_cgxmapped(rvu, pf)) { 4058 /* Get CGX and LMAC to which this PF is mapped and find link */ 4059 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx, &lmac); 4060 link = (cgx * hw->lmac_per_cgx) + lmac; 4061 } else if (pf == 0) { 4062 /* For VFs of PF0 ingress is LBK port, so config LBK link */ 4063 pfvf = rvu_get_pfvf(rvu, pcifunc); 4064 link = hw->cgx_links + pfvf->lbkid; 4065 } 4066 4067 if (link < 0) 4068 return NIX_AF_ERR_RX_LINK_INVALID; 4069 4070 nix_find_link_frs(rvu, req, pcifunc); 4071 4072 linkcfg: 4073 cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link)); 4074 cfg = (cfg & ~(0xFFFFULL << 16)) | ((u64)req->maxlen << 16); 4075 if (req->update_minlen) 4076 cfg = (cfg & ~0xFFFFULL) | req->minlen; 4077 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), cfg); 4078 4079 if (req->sdp_link || pf == 0) 4080 return 0; 4081 4082 /* Update transmit credits for CGX links */ 4083 lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, lmac); 4084 if (!lmac_fifo_len) { 4085 dev_err(rvu->dev, 4086 "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", 4087 __func__, cgx, lmac); 4088 return 0; 4089 } 4090 return nix_config_link_credits(rvu, blkaddr, link, pcifunc, 4091 (lmac_fifo_len - req->maxlen) / 16); 4092 } 4093 4094 int rvu_mbox_handler_nix_set_rx_cfg(struct rvu *rvu, struct nix_rx_cfg *req, 4095 struct msg_rsp *rsp) 4096 { 4097 int nixlf, blkaddr, err; 4098 u64 cfg; 4099 4100 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &nixlf, &blkaddr); 4101 if (err) 4102 return err; 4103 4104 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf)); 4105 /* Set the interface configuration */ 4106 if (req->len_verify & BIT(0)) 4107 cfg |= BIT_ULL(41); 4108 else 4109 cfg &= ~BIT_ULL(41); 4110 4111 if (req->len_verify & BIT(1)) 4112 cfg |= BIT_ULL(40); 4113 else 4114 cfg &= ~BIT_ULL(40); 4115 4116 if (req->csum_verify & BIT(0)) 4117 cfg |= BIT_ULL(37); 4118 else 4119 cfg &= ~BIT_ULL(37); 4120 4121 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_CFG(nixlf), cfg); 4122 4123 return 0; 4124 } 4125 4126 static u64 rvu_get_lbk_link_credits(struct rvu *rvu, u16 lbk_max_frs) 4127 { 4128 /* CN10k supports 72KB FIFO size and max packet size of 64k */ 4129 if (rvu->hw->lbk_bufsize == 0x12000) 4130 return (rvu->hw->lbk_bufsize - lbk_max_frs) / 16; 4131 4132 return 1600; /* 16 * max LBK datarate = 16 * 100Gbps */ 4133 } 4134 4135 static void nix_link_config(struct rvu *rvu, int blkaddr, 4136 struct nix_hw *nix_hw) 4137 { 4138 struct rvu_hwinfo *hw = rvu->hw; 4139 int cgx, lmac_cnt, slink, link; 4140 u16 lbk_max_frs, lmac_max_frs; 4141 unsigned long lmac_bmap; 4142 u64 tx_credits, cfg; 4143 u64 lmac_fifo_len; 4144 int iter; 4145 4146 rvu_get_lbk_link_max_frs(rvu, &lbk_max_frs); 4147 rvu_get_lmac_link_max_frs(rvu, &lmac_max_frs); 4148 4149 /* Set default min/max packet lengths allowed on NIX Rx links. 4150 * 4151 * With HW reset minlen value of 60byte, HW will treat ARP pkts 4152 * as undersize and report them to SW as error pkts, hence 4153 * setting it to 40 bytes. 4154 */ 4155 for (link = 0; link < hw->cgx_links; link++) { 4156 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4157 ((u64)lmac_max_frs << 16) | NIC_HW_MIN_FRS); 4158 } 4159 4160 for (link = hw->cgx_links; link < hw->lbk_links; link++) { 4161 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4162 ((u64)lbk_max_frs << 16) | NIC_HW_MIN_FRS); 4163 } 4164 if (hw->sdp_links) { 4165 link = hw->cgx_links + hw->lbk_links; 4166 rvu_write64(rvu, blkaddr, NIX_AF_RX_LINKX_CFG(link), 4167 SDP_HW_MAX_FRS << 16 | NIC_HW_MIN_FRS); 4168 } 4169 4170 /* Set credits for Tx links assuming max packet length allowed. 4171 * This will be reconfigured based on MTU set for PF/VF. 4172 */ 4173 for (cgx = 0; cgx < hw->cgx; cgx++) { 4174 lmac_cnt = cgx_get_lmac_cnt(rvu_cgx_pdata(cgx, rvu)); 4175 /* Skip when cgx is not available or lmac cnt is zero */ 4176 if (lmac_cnt <= 0) 4177 continue; 4178 slink = cgx * hw->lmac_per_cgx; 4179 4180 /* Get LMAC id's from bitmap */ 4181 lmac_bmap = cgx_get_lmac_bmap(rvu_cgx_pdata(cgx, rvu)); 4182 for_each_set_bit(iter, &lmac_bmap, rvu->hw->lmac_per_cgx) { 4183 lmac_fifo_len = rvu_cgx_get_lmac_fifolen(rvu, cgx, iter); 4184 if (!lmac_fifo_len) { 4185 dev_err(rvu->dev, 4186 "%s: Failed to get CGX/RPM%d:LMAC%d FIFO size\n", 4187 __func__, cgx, iter); 4188 continue; 4189 } 4190 tx_credits = (lmac_fifo_len - lmac_max_frs) / 16; 4191 /* Enable credits and set credit pkt count to max allowed */ 4192 cfg = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4193 4194 link = iter + slink; 4195 nix_hw->tx_credits[link] = tx_credits; 4196 rvu_write64(rvu, blkaddr, 4197 NIX_AF_TX_LINKX_NORM_CREDIT(link), cfg); 4198 } 4199 } 4200 4201 /* Set Tx credits for LBK link */ 4202 slink = hw->cgx_links; 4203 for (link = slink; link < (slink + hw->lbk_links); link++) { 4204 tx_credits = rvu_get_lbk_link_credits(rvu, lbk_max_frs); 4205 nix_hw->tx_credits[link] = tx_credits; 4206 /* Enable credits and set credit pkt count to max allowed */ 4207 tx_credits = (tx_credits << 12) | (0x1FF << 2) | BIT_ULL(1); 4208 rvu_write64(rvu, blkaddr, 4209 NIX_AF_TX_LINKX_NORM_CREDIT(link), tx_credits); 4210 } 4211 } 4212 4213 static int nix_calibrate_x2p(struct rvu *rvu, int blkaddr) 4214 { 4215 int idx, err; 4216 u64 status; 4217 4218 /* Start X2P bus calibration */ 4219 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4220 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | BIT_ULL(9)); 4221 /* Wait for calibration to complete */ 4222 err = rvu_poll_reg(rvu, blkaddr, 4223 NIX_AF_STATUS, BIT_ULL(10), false); 4224 if (err) { 4225 dev_err(rvu->dev, "NIX X2P bus calibration failed\n"); 4226 return err; 4227 } 4228 4229 status = rvu_read64(rvu, blkaddr, NIX_AF_STATUS); 4230 /* Check if CGX devices are ready */ 4231 for (idx = 0; idx < rvu->cgx_cnt_max; idx++) { 4232 /* Skip when cgx port is not available */ 4233 if (!rvu_cgx_pdata(idx, rvu) || 4234 (status & (BIT_ULL(16 + idx)))) 4235 continue; 4236 dev_err(rvu->dev, 4237 "CGX%d didn't respond to NIX X2P calibration\n", idx); 4238 err = -EBUSY; 4239 } 4240 4241 /* Check if LBK is ready */ 4242 if (!(status & BIT_ULL(19))) { 4243 dev_err(rvu->dev, 4244 "LBK didn't respond to NIX X2P calibration\n"); 4245 err = -EBUSY; 4246 } 4247 4248 /* Clear 'calibrate_x2p' bit */ 4249 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4250 rvu_read64(rvu, blkaddr, NIX_AF_CFG) & ~BIT_ULL(9)); 4251 if (err || (status & 0x3FFULL)) 4252 dev_err(rvu->dev, 4253 "NIX X2P calibration failed, status 0x%llx\n", status); 4254 if (err) 4255 return err; 4256 return 0; 4257 } 4258 4259 static int nix_aq_init(struct rvu *rvu, struct rvu_block *block) 4260 { 4261 u64 cfg; 4262 int err; 4263 4264 /* Set admin queue endianness */ 4265 cfg = rvu_read64(rvu, block->addr, NIX_AF_CFG); 4266 #ifdef __BIG_ENDIAN 4267 cfg |= BIT_ULL(8); 4268 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4269 #else 4270 cfg &= ~BIT_ULL(8); 4271 rvu_write64(rvu, block->addr, NIX_AF_CFG, cfg); 4272 #endif 4273 4274 /* Do not bypass NDC cache */ 4275 cfg = rvu_read64(rvu, block->addr, NIX_AF_NDC_CFG); 4276 cfg &= ~0x3FFEULL; 4277 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 4278 /* Disable caching of SQB aka SQEs */ 4279 cfg |= 0x04ULL; 4280 #endif 4281 rvu_write64(rvu, block->addr, NIX_AF_NDC_CFG, cfg); 4282 4283 /* Result structure can be followed by RQ/SQ/CQ context at 4284 * RES + 128bytes and a write mask at RES + 256 bytes, depending on 4285 * operation type. Alloc sufficient result memory for all operations. 4286 */ 4287 err = rvu_aq_alloc(rvu, &block->aq, 4288 Q_COUNT(AQ_SIZE), sizeof(struct nix_aq_inst_s), 4289 ALIGN(sizeof(struct nix_aq_res_s), 128) + 256); 4290 if (err) 4291 return err; 4292 4293 rvu_write64(rvu, block->addr, NIX_AF_AQ_CFG, AQ_SIZE); 4294 rvu_write64(rvu, block->addr, 4295 NIX_AF_AQ_BASE, (u64)block->aq->inst->iova); 4296 return 0; 4297 } 4298 4299 static void rvu_nix_setup_capabilities(struct rvu *rvu, int blkaddr) 4300 { 4301 struct rvu_hwinfo *hw = rvu->hw; 4302 u64 hw_const; 4303 4304 hw_const = rvu_read64(rvu, blkaddr, NIX_AF_CONST1); 4305 4306 /* On OcteonTx2 DWRR quantum is directly configured into each of 4307 * the transmit scheduler queues. And PF/VF drivers were free to 4308 * config any value upto 2^24. 4309 * On CN10K, HW is modified, the quantum configuration at scheduler 4310 * queues is in terms of weight. And SW needs to setup a base DWRR MTU 4311 * at NIX_AF_DWRR_RPM_MTU / NIX_AF_DWRR_SDP_MTU. HW will do 4312 * 'DWRR MTU * weight' to get the quantum. 4313 * 4314 * Check if HW uses a common MTU for all DWRR quantum configs. 4315 * On OcteonTx2 this register field is '0'. 4316 */ 4317 if (((hw_const >> 56) & 0x10) == 0x10) 4318 hw->cap.nix_common_dwrr_mtu = true; 4319 } 4320 4321 static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw) 4322 { 4323 const struct npc_lt_def_cfg *ltdefs; 4324 struct rvu_hwinfo *hw = rvu->hw; 4325 int blkaddr = nix_hw->blkaddr; 4326 struct rvu_block *block; 4327 int err; 4328 u64 cfg; 4329 4330 block = &hw->block[blkaddr]; 4331 4332 if (is_rvu_96xx_B0(rvu)) { 4333 /* As per a HW errata in 96xx A0/B0 silicon, NIX may corrupt 4334 * internal state when conditional clocks are turned off. 4335 * Hence enable them. 4336 */ 4337 rvu_write64(rvu, blkaddr, NIX_AF_CFG, 4338 rvu_read64(rvu, blkaddr, NIX_AF_CFG) | 0x40ULL); 4339 4340 /* Set chan/link to backpressure TL3 instead of TL2 */ 4341 rvu_write64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL, 0x01); 4342 4343 /* Disable SQ manager's sticky mode operation (set TM6 = 0) 4344 * This sticky mode is known to cause SQ stalls when multiple 4345 * SQs are mapped to same SMQ and transmitting pkts at a time. 4346 */ 4347 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS); 4348 cfg &= ~BIT_ULL(15); 4349 rvu_write64(rvu, blkaddr, NIX_AF_SQM_DBG_CTL_STATUS, cfg); 4350 } 4351 4352 ltdefs = rvu->kpu.lt_def; 4353 /* Calibrate X2P bus to check if CGX/LBK links are fine */ 4354 err = nix_calibrate_x2p(rvu, blkaddr); 4355 if (err) 4356 return err; 4357 4358 /* Setup capabilities of the NIX block */ 4359 rvu_nix_setup_capabilities(rvu, blkaddr); 4360 4361 /* Initialize admin queue */ 4362 err = nix_aq_init(rvu, block); 4363 if (err) 4364 return err; 4365 4366 /* Restore CINT timer delay to HW reset values */ 4367 rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL); 4368 4369 cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG); 4370 4371 /* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */ 4372 cfg |= 1ULL; 4373 if (!is_rvu_otx2(rvu)) 4374 cfg |= NIX_PTP_1STEP_EN; 4375 4376 rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg); 4377 4378 if (!is_rvu_otx2(rvu)) 4379 rvu_nix_block_cn10k_init(rvu, nix_hw); 4380 4381 if (is_block_implemented(hw, blkaddr)) { 4382 err = nix_setup_txschq(rvu, nix_hw, blkaddr); 4383 if (err) 4384 return err; 4385 4386 err = nix_setup_ipolicers(rvu, nix_hw, blkaddr); 4387 if (err) 4388 return err; 4389 4390 err = nix_af_mark_format_setup(rvu, nix_hw, blkaddr); 4391 if (err) 4392 return err; 4393 4394 err = nix_setup_mcast(rvu, nix_hw, blkaddr); 4395 if (err) 4396 return err; 4397 4398 err = nix_setup_txvlan(rvu, nix_hw); 4399 if (err) 4400 return err; 4401 4402 /* Configure segmentation offload formats */ 4403 nix_setup_lso(rvu, nix_hw, blkaddr); 4404 4405 /* Config Outer/Inner L2, IP, TCP, UDP and SCTP NPC layer info. 4406 * This helps HW protocol checker to identify headers 4407 * and validate length and checksums. 4408 */ 4409 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OL2, 4410 (ltdefs->rx_ol2.lid << 8) | (ltdefs->rx_ol2.ltype_match << 4) | 4411 ltdefs->rx_ol2.ltype_mask); 4412 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4, 4413 (ltdefs->rx_oip4.lid << 8) | (ltdefs->rx_oip4.ltype_match << 4) | 4414 ltdefs->rx_oip4.ltype_mask); 4415 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4, 4416 (ltdefs->rx_iip4.lid << 8) | (ltdefs->rx_iip4.ltype_match << 4) | 4417 ltdefs->rx_iip4.ltype_mask); 4418 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6, 4419 (ltdefs->rx_oip6.lid << 8) | (ltdefs->rx_oip6.ltype_match << 4) | 4420 ltdefs->rx_oip6.ltype_mask); 4421 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6, 4422 (ltdefs->rx_iip6.lid << 8) | (ltdefs->rx_iip6.ltype_match << 4) | 4423 ltdefs->rx_iip6.ltype_mask); 4424 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OTCP, 4425 (ltdefs->rx_otcp.lid << 8) | (ltdefs->rx_otcp.ltype_match << 4) | 4426 ltdefs->rx_otcp.ltype_mask); 4427 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ITCP, 4428 (ltdefs->rx_itcp.lid << 8) | (ltdefs->rx_itcp.ltype_match << 4) | 4429 ltdefs->rx_itcp.ltype_mask); 4430 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OUDP, 4431 (ltdefs->rx_oudp.lid << 8) | (ltdefs->rx_oudp.ltype_match << 4) | 4432 ltdefs->rx_oudp.ltype_mask); 4433 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IUDP, 4434 (ltdefs->rx_iudp.lid << 8) | (ltdefs->rx_iudp.ltype_match << 4) | 4435 ltdefs->rx_iudp.ltype_mask); 4436 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OSCTP, 4437 (ltdefs->rx_osctp.lid << 8) | (ltdefs->rx_osctp.ltype_match << 4) | 4438 ltdefs->rx_osctp.ltype_mask); 4439 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ISCTP, 4440 (ltdefs->rx_isctp.lid << 8) | (ltdefs->rx_isctp.ltype_match << 4) | 4441 ltdefs->rx_isctp.ltype_mask); 4442 4443 if (!is_rvu_otx2(rvu)) { 4444 /* Enable APAD calculation for other protocols 4445 * matching APAD0 and APAD1 lt def registers. 4446 */ 4447 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD0, 4448 (ltdefs->rx_apad0.valid << 11) | 4449 (ltdefs->rx_apad0.lid << 8) | 4450 (ltdefs->rx_apad0.ltype_match << 4) | 4451 ltdefs->rx_apad0.ltype_mask); 4452 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_CST_APAD1, 4453 (ltdefs->rx_apad1.valid << 11) | 4454 (ltdefs->rx_apad1.lid << 8) | 4455 (ltdefs->rx_apad1.ltype_match << 4) | 4456 ltdefs->rx_apad1.ltype_mask); 4457 4458 /* Receive ethertype defination register defines layer 4459 * information in NPC_RESULT_S to identify the Ethertype 4460 * location in L2 header. Used for Ethertype overwriting 4461 * in inline IPsec flow. 4462 */ 4463 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(0), 4464 (ltdefs->rx_et[0].offset << 12) | 4465 (ltdefs->rx_et[0].valid << 11) | 4466 (ltdefs->rx_et[0].lid << 8) | 4467 (ltdefs->rx_et[0].ltype_match << 4) | 4468 ltdefs->rx_et[0].ltype_mask); 4469 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_ET(1), 4470 (ltdefs->rx_et[1].offset << 12) | 4471 (ltdefs->rx_et[1].valid << 11) | 4472 (ltdefs->rx_et[1].lid << 8) | 4473 (ltdefs->rx_et[1].ltype_match << 4) | 4474 ltdefs->rx_et[1].ltype_mask); 4475 } 4476 4477 err = nix_rx_flowkey_alg_cfg(rvu, blkaddr); 4478 if (err) 4479 return err; 4480 4481 nix_hw->tx_credits = kcalloc(hw->cgx_links + hw->lbk_links, 4482 sizeof(u64), GFP_KERNEL); 4483 if (!nix_hw->tx_credits) 4484 return -ENOMEM; 4485 4486 /* Initialize CGX/LBK/SDP link credits, min/max pkt lengths */ 4487 nix_link_config(rvu, blkaddr, nix_hw); 4488 4489 /* Enable Channel backpressure */ 4490 rvu_write64(rvu, blkaddr, NIX_AF_RX_CFG, BIT_ULL(0)); 4491 } 4492 return 0; 4493 } 4494 4495 int rvu_nix_init(struct rvu *rvu) 4496 { 4497 struct rvu_hwinfo *hw = rvu->hw; 4498 struct nix_hw *nix_hw; 4499 int blkaddr = 0, err; 4500 int i = 0; 4501 4502 hw->nix = devm_kcalloc(rvu->dev, MAX_NIX_BLKS, sizeof(struct nix_hw), 4503 GFP_KERNEL); 4504 if (!hw->nix) 4505 return -ENOMEM; 4506 4507 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 4508 while (blkaddr) { 4509 nix_hw = &hw->nix[i]; 4510 nix_hw->rvu = rvu; 4511 nix_hw->blkaddr = blkaddr; 4512 err = rvu_nix_block_init(rvu, nix_hw); 4513 if (err) 4514 return err; 4515 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 4516 i++; 4517 } 4518 4519 return 0; 4520 } 4521 4522 static void rvu_nix_block_freemem(struct rvu *rvu, int blkaddr, 4523 struct rvu_block *block) 4524 { 4525 struct nix_txsch *txsch; 4526 struct nix_mcast *mcast; 4527 struct nix_txvlan *vlan; 4528 struct nix_hw *nix_hw; 4529 int lvl; 4530 4531 rvu_aq_free(rvu, block->aq); 4532 4533 if (is_block_implemented(rvu->hw, blkaddr)) { 4534 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4535 if (!nix_hw) 4536 return; 4537 4538 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { 4539 txsch = &nix_hw->txsch[lvl]; 4540 kfree(txsch->schq.bmap); 4541 } 4542 4543 kfree(nix_hw->tx_credits); 4544 4545 nix_ipolicer_freemem(rvu, nix_hw); 4546 4547 vlan = &nix_hw->txvlan; 4548 kfree(vlan->rsrc.bmap); 4549 mutex_destroy(&vlan->rsrc_lock); 4550 4551 mcast = &nix_hw->mcast; 4552 qmem_free(rvu->dev, mcast->mce_ctx); 4553 qmem_free(rvu->dev, mcast->mcast_buf); 4554 mutex_destroy(&mcast->mce_lock); 4555 } 4556 } 4557 4558 void rvu_nix_freemem(struct rvu *rvu) 4559 { 4560 struct rvu_hwinfo *hw = rvu->hw; 4561 struct rvu_block *block; 4562 int blkaddr = 0; 4563 4564 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 4565 while (blkaddr) { 4566 block = &hw->block[blkaddr]; 4567 rvu_nix_block_freemem(rvu, blkaddr, block); 4568 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr); 4569 } 4570 } 4571 4572 int rvu_mbox_handler_nix_lf_start_rx(struct rvu *rvu, struct msg_req *req, 4573 struct msg_rsp *rsp) 4574 { 4575 u16 pcifunc = req->hdr.pcifunc; 4576 struct rvu_pfvf *pfvf; 4577 int nixlf, err; 4578 4579 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4580 if (err) 4581 return err; 4582 4583 rvu_npc_enable_default_entries(rvu, pcifunc, nixlf); 4584 4585 npc_mcam_enable_flows(rvu, pcifunc); 4586 4587 pfvf = rvu_get_pfvf(rvu, pcifunc); 4588 set_bit(NIXLF_INITIALIZED, &pfvf->flags); 4589 4590 rvu_switch_update_rules(rvu, pcifunc); 4591 4592 return rvu_cgx_start_stop_io(rvu, pcifunc, true); 4593 } 4594 4595 int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, 4596 struct msg_rsp *rsp) 4597 { 4598 u16 pcifunc = req->hdr.pcifunc; 4599 struct rvu_pfvf *pfvf; 4600 int nixlf, err; 4601 4602 err = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); 4603 if (err) 4604 return err; 4605 4606 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 4607 4608 pfvf = rvu_get_pfvf(rvu, pcifunc); 4609 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 4610 4611 return rvu_cgx_start_stop_io(rvu, pcifunc, false); 4612 } 4613 4614 #define RX_SA_BASE GENMASK_ULL(52, 7) 4615 4616 void rvu_nix_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int nixlf) 4617 { 4618 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 4619 struct hwctx_disable_req ctx_req; 4620 int pf = rvu_get_pf(pcifunc); 4621 struct mac_ops *mac_ops; 4622 u8 cgx_id, lmac_id; 4623 u64 sa_base; 4624 void *cgxd; 4625 int err; 4626 4627 ctx_req.hdr.pcifunc = pcifunc; 4628 4629 /* Cleanup NPC MCAM entries, free Tx scheduler queues being used */ 4630 rvu_npc_disable_mcam_entries(rvu, pcifunc, nixlf); 4631 rvu_npc_free_mcam_entries(rvu, pcifunc, nixlf); 4632 nix_interface_deinit(rvu, pcifunc, nixlf); 4633 nix_rx_sync(rvu, blkaddr); 4634 nix_txschq_free(rvu, pcifunc); 4635 4636 clear_bit(NIXLF_INITIALIZED, &pfvf->flags); 4637 4638 rvu_cgx_start_stop_io(rvu, pcifunc, false); 4639 4640 if (pfvf->sq_ctx) { 4641 ctx_req.ctype = NIX_AQ_CTYPE_SQ; 4642 err = nix_lf_hwctx_disable(rvu, &ctx_req); 4643 if (err) 4644 dev_err(rvu->dev, "SQ ctx disable failed\n"); 4645 } 4646 4647 if (pfvf->rq_ctx) { 4648 ctx_req.ctype = NIX_AQ_CTYPE_RQ; 4649 err = nix_lf_hwctx_disable(rvu, &ctx_req); 4650 if (err) 4651 dev_err(rvu->dev, "RQ ctx disable failed\n"); 4652 } 4653 4654 if (pfvf->cq_ctx) { 4655 ctx_req.ctype = NIX_AQ_CTYPE_CQ; 4656 err = nix_lf_hwctx_disable(rvu, &ctx_req); 4657 if (err) 4658 dev_err(rvu->dev, "CQ ctx disable failed\n"); 4659 } 4660 4661 /* reset HW config done for Switch headers */ 4662 rvu_npc_set_parse_mode(rvu, pcifunc, OTX2_PRIV_FLAGS_DEFAULT, 4663 (PKIND_TX | PKIND_RX), 0, 0, 0, 0); 4664 4665 /* Disabling CGX and NPC config done for PTP */ 4666 if (pfvf->hw_rx_tstamp_en) { 4667 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id); 4668 cgxd = rvu_cgx_pdata(cgx_id, rvu); 4669 mac_ops = get_mac_ops(cgxd); 4670 mac_ops->mac_enadis_ptp_config(cgxd, lmac_id, false); 4671 /* Undo NPC config done for PTP */ 4672 if (npc_config_ts_kpuaction(rvu, pf, pcifunc, false)) 4673 dev_err(rvu->dev, "NPC config for PTP failed\n"); 4674 pfvf->hw_rx_tstamp_en = false; 4675 } 4676 4677 /* reset priority flow control config */ 4678 rvu_cgx_prio_flow_ctrl_cfg(rvu, pcifunc, 0, 0, 0); 4679 4680 /* reset 802.3x flow control config */ 4681 rvu_cgx_cfg_pause_frm(rvu, pcifunc, 0, 0); 4682 4683 nix_ctx_free(rvu, pfvf); 4684 4685 nix_free_all_bandprof(rvu, pcifunc); 4686 4687 sa_base = rvu_read64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(nixlf)); 4688 if (FIELD_GET(RX_SA_BASE, sa_base)) { 4689 err = rvu_cpt_ctx_flush(rvu, pcifunc); 4690 if (err) 4691 dev_err(rvu->dev, 4692 "CPT ctx flush failed with error: %d\n", err); 4693 } 4694 } 4695 4696 #define NIX_AF_LFX_TX_CFG_PTP_EN BIT_ULL(32) 4697 4698 static int rvu_nix_lf_ptp_tx_cfg(struct rvu *rvu, u16 pcifunc, bool enable) 4699 { 4700 struct rvu_hwinfo *hw = rvu->hw; 4701 struct rvu_block *block; 4702 int blkaddr, pf; 4703 int nixlf; 4704 u64 cfg; 4705 4706 pf = rvu_get_pf(pcifunc); 4707 if (!is_mac_feature_supported(rvu, pf, RVU_LMAC_FEAT_PTP)) 4708 return 0; 4709 4710 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 4711 if (blkaddr < 0) 4712 return NIX_AF_ERR_AF_LF_INVALID; 4713 4714 block = &hw->block[blkaddr]; 4715 nixlf = rvu_get_lf(rvu, block, pcifunc, 0); 4716 if (nixlf < 0) 4717 return NIX_AF_ERR_AF_LF_INVALID; 4718 4719 cfg = rvu_read64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf)); 4720 4721 if (enable) 4722 cfg |= NIX_AF_LFX_TX_CFG_PTP_EN; 4723 else 4724 cfg &= ~NIX_AF_LFX_TX_CFG_PTP_EN; 4725 4726 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_CFG(nixlf), cfg); 4727 4728 return 0; 4729 } 4730 4731 int rvu_mbox_handler_nix_lf_ptp_tx_enable(struct rvu *rvu, struct msg_req *req, 4732 struct msg_rsp *rsp) 4733 { 4734 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, true); 4735 } 4736 4737 int rvu_mbox_handler_nix_lf_ptp_tx_disable(struct rvu *rvu, struct msg_req *req, 4738 struct msg_rsp *rsp) 4739 { 4740 return rvu_nix_lf_ptp_tx_cfg(rvu, req->hdr.pcifunc, false); 4741 } 4742 4743 int rvu_mbox_handler_nix_lso_format_cfg(struct rvu *rvu, 4744 struct nix_lso_format_cfg *req, 4745 struct nix_lso_format_cfg_rsp *rsp) 4746 { 4747 u16 pcifunc = req->hdr.pcifunc; 4748 struct nix_hw *nix_hw; 4749 struct rvu_pfvf *pfvf; 4750 int blkaddr, idx, f; 4751 u64 reg; 4752 4753 pfvf = rvu_get_pfvf(rvu, pcifunc); 4754 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 4755 if (!pfvf->nixlf || blkaddr < 0) 4756 return NIX_AF_ERR_AF_LF_INVALID; 4757 4758 nix_hw = get_nix_hw(rvu->hw, blkaddr); 4759 if (!nix_hw) 4760 return NIX_AF_ERR_INVALID_NIXBLK; 4761 4762 /* Find existing matching LSO format, if any */ 4763 for (idx = 0; idx < nix_hw->lso.in_use; idx++) { 4764 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) { 4765 reg = rvu_read64(rvu, blkaddr, 4766 NIX_AF_LSO_FORMATX_FIELDX(idx, f)); 4767 if (req->fields[f] != (reg & req->field_mask)) 4768 break; 4769 } 4770 4771 if (f == NIX_LSO_FIELD_MAX) 4772 break; 4773 } 4774 4775 if (idx < nix_hw->lso.in_use) { 4776 /* Match found */ 4777 rsp->lso_format_idx = idx; 4778 return 0; 4779 } 4780 4781 if (nix_hw->lso.in_use == nix_hw->lso.total) 4782 return NIX_AF_ERR_LSO_CFG_FAIL; 4783 4784 rsp->lso_format_idx = nix_hw->lso.in_use++; 4785 4786 for (f = 0; f < NIX_LSO_FIELD_MAX; f++) 4787 rvu_write64(rvu, blkaddr, 4788 NIX_AF_LSO_FORMATX_FIELDX(rsp->lso_format_idx, f), 4789 req->fields[f]); 4790 4791 return 0; 4792 } 4793 4794 #define IPSEC_GEN_CFG_EGRP GENMASK_ULL(50, 48) 4795 #define IPSEC_GEN_CFG_OPCODE GENMASK_ULL(47, 32) 4796 #define IPSEC_GEN_CFG_PARAM1 GENMASK_ULL(31, 16) 4797 #define IPSEC_GEN_CFG_PARAM2 GENMASK_ULL(15, 0) 4798 4799 #define CPT_INST_QSEL_BLOCK GENMASK_ULL(28, 24) 4800 #define CPT_INST_QSEL_PF_FUNC GENMASK_ULL(23, 8) 4801 #define CPT_INST_QSEL_SLOT GENMASK_ULL(7, 0) 4802 4803 #define CPT_INST_CREDIT_TH GENMASK_ULL(53, 32) 4804 #define CPT_INST_CREDIT_BPID GENMASK_ULL(30, 22) 4805 #define CPT_INST_CREDIT_CNT GENMASK_ULL(21, 0) 4806 4807 static void nix_inline_ipsec_cfg(struct rvu *rvu, struct nix_inline_ipsec_cfg *req, 4808 int blkaddr) 4809 { 4810 u8 cpt_idx, cpt_blkaddr; 4811 u64 val; 4812 4813 cpt_idx = (blkaddr == BLKADDR_NIX0) ? 0 : 1; 4814 if (req->enable) { 4815 val = 0; 4816 /* Enable context prefetching */ 4817 if (!is_rvu_otx2(rvu)) 4818 val |= BIT_ULL(51); 4819 4820 /* Set OPCODE and EGRP */ 4821 val |= FIELD_PREP(IPSEC_GEN_CFG_EGRP, req->gen_cfg.egrp); 4822 val |= FIELD_PREP(IPSEC_GEN_CFG_OPCODE, req->gen_cfg.opcode); 4823 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM1, req->gen_cfg.param1); 4824 val |= FIELD_PREP(IPSEC_GEN_CFG_PARAM2, req->gen_cfg.param2); 4825 4826 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, val); 4827 4828 /* Set CPT queue for inline IPSec */ 4829 val = FIELD_PREP(CPT_INST_QSEL_SLOT, req->inst_qsel.cpt_slot); 4830 val |= FIELD_PREP(CPT_INST_QSEL_PF_FUNC, 4831 req->inst_qsel.cpt_pf_func); 4832 4833 if (!is_rvu_otx2(rvu)) { 4834 cpt_blkaddr = (cpt_idx == 0) ? BLKADDR_CPT0 : 4835 BLKADDR_CPT1; 4836 val |= FIELD_PREP(CPT_INST_QSEL_BLOCK, cpt_blkaddr); 4837 } 4838 4839 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 4840 val); 4841 4842 /* Set CPT credit */ 4843 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 4844 if ((val & 0x3FFFFF) != 0x3FFFFF) 4845 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 4846 0x3FFFFF - val); 4847 4848 val = FIELD_PREP(CPT_INST_CREDIT_CNT, req->cpt_credit); 4849 val |= FIELD_PREP(CPT_INST_CREDIT_BPID, req->bpid); 4850 val |= FIELD_PREP(CPT_INST_CREDIT_TH, req->credit_th); 4851 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), val); 4852 } else { 4853 rvu_write64(rvu, blkaddr, NIX_AF_RX_IPSEC_GEN_CFG, 0x0); 4854 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_INST_QSEL(cpt_idx), 4855 0x0); 4856 val = rvu_read64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx)); 4857 if ((val & 0x3FFFFF) != 0x3FFFFF) 4858 rvu_write64(rvu, blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 4859 0x3FFFFF - val); 4860 } 4861 } 4862 4863 int rvu_mbox_handler_nix_inline_ipsec_cfg(struct rvu *rvu, 4864 struct nix_inline_ipsec_cfg *req, 4865 struct msg_rsp *rsp) 4866 { 4867 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 4868 return 0; 4869 4870 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX0); 4871 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 4872 nix_inline_ipsec_cfg(rvu, req, BLKADDR_NIX1); 4873 4874 return 0; 4875 } 4876 4877 int rvu_mbox_handler_nix_read_inline_ipsec_cfg(struct rvu *rvu, 4878 struct msg_req *req, 4879 struct nix_inline_ipsec_cfg *rsp) 4880 4881 { 4882 u64 val; 4883 4884 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 4885 return 0; 4886 4887 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_IPSEC_GEN_CFG); 4888 rsp->gen_cfg.egrp = FIELD_GET(IPSEC_GEN_CFG_EGRP, val); 4889 rsp->gen_cfg.opcode = FIELD_GET(IPSEC_GEN_CFG_OPCODE, val); 4890 rsp->gen_cfg.param1 = FIELD_GET(IPSEC_GEN_CFG_PARAM1, val); 4891 rsp->gen_cfg.param2 = FIELD_GET(IPSEC_GEN_CFG_PARAM2, val); 4892 4893 val = rvu_read64(rvu, BLKADDR_NIX0, NIX_AF_RX_CPTX_CREDIT(0)); 4894 rsp->cpt_credit = FIELD_GET(CPT_INST_CREDIT_CNT, val); 4895 rsp->credit_th = FIELD_GET(CPT_INST_CREDIT_TH, val); 4896 rsp->bpid = FIELD_GET(CPT_INST_CREDIT_BPID, val); 4897 4898 return 0; 4899 } 4900 4901 int rvu_mbox_handler_nix_inline_ipsec_lf_cfg(struct rvu *rvu, 4902 struct nix_inline_ipsec_lf_cfg *req, 4903 struct msg_rsp *rsp) 4904 { 4905 int lf, blkaddr, err; 4906 u64 val; 4907 4908 if (!is_block_implemented(rvu->hw, BLKADDR_CPT0)) 4909 return 0; 4910 4911 err = nix_get_nixlf(rvu, req->hdr.pcifunc, &lf, &blkaddr); 4912 if (err) 4913 return err; 4914 4915 if (req->enable) { 4916 /* Set TT, TAG_CONST, SA_POW2_SIZE and LENM1_MAX */ 4917 val = (u64)req->ipsec_cfg0.tt << 44 | 4918 (u64)req->ipsec_cfg0.tag_const << 20 | 4919 (u64)req->ipsec_cfg0.sa_pow2_size << 16 | 4920 req->ipsec_cfg0.lenm1_max; 4921 4922 if (blkaddr == BLKADDR_NIX1) 4923 val |= BIT_ULL(46); 4924 4925 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), val); 4926 4927 /* Set SA_IDX_W and SA_IDX_MAX */ 4928 val = (u64)req->ipsec_cfg1.sa_idx_w << 32 | 4929 req->ipsec_cfg1.sa_idx_max; 4930 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), val); 4931 4932 /* Set SA base address */ 4933 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 4934 req->sa_base_addr); 4935 } else { 4936 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG0(lf), 0x0); 4937 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_CFG1(lf), 0x0); 4938 rvu_write64(rvu, blkaddr, NIX_AF_LFX_RX_IPSEC_SA_BASE(lf), 4939 0x0); 4940 } 4941 4942 return 0; 4943 } 4944 4945 void rvu_nix_reset_mac(struct rvu_pfvf *pfvf, int pcifunc) 4946 { 4947 bool from_vf = !!(pcifunc & RVU_PFVF_FUNC_MASK); 4948 4949 /* overwrite vf mac address with default_mac */ 4950 if (from_vf) 4951 ether_addr_copy(pfvf->mac_addr, pfvf->default_mac); 4952 } 4953 4954 /* NIX ingress policers or bandwidth profiles APIs */ 4955 static void nix_config_rx_pkt_policer_precolor(struct rvu *rvu, int blkaddr) 4956 { 4957 struct npc_lt_def_cfg defs, *ltdefs; 4958 4959 ltdefs = &defs; 4960 memcpy(ltdefs, rvu->kpu.lt_def, sizeof(struct npc_lt_def_cfg)); 4961 4962 /* Extract PCP and DEI fields from outer VLAN from byte offset 4963 * 2 from the start of LB_PTR (ie TAG). 4964 * VLAN0 is Outer VLAN and VLAN1 is Inner VLAN. Inner VLAN 4965 * fields are considered when 'Tunnel enable' is set in profile. 4966 */ 4967 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN0_PCP_DEI, 4968 (2UL << 12) | (ltdefs->ovlan.lid << 8) | 4969 (ltdefs->ovlan.ltype_match << 4) | 4970 ltdefs->ovlan.ltype_mask); 4971 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_VLAN1_PCP_DEI, 4972 (2UL << 12) | (ltdefs->ivlan.lid << 8) | 4973 (ltdefs->ivlan.ltype_match << 4) | 4974 ltdefs->ivlan.ltype_mask); 4975 4976 /* DSCP field in outer and tunneled IPv4 packets */ 4977 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP4_DSCP, 4978 (1UL << 12) | (ltdefs->rx_oip4.lid << 8) | 4979 (ltdefs->rx_oip4.ltype_match << 4) | 4980 ltdefs->rx_oip4.ltype_mask); 4981 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP4_DSCP, 4982 (1UL << 12) | (ltdefs->rx_iip4.lid << 8) | 4983 (ltdefs->rx_iip4.ltype_match << 4) | 4984 ltdefs->rx_iip4.ltype_mask); 4985 4986 /* DSCP field (traffic class) in outer and tunneled IPv6 packets */ 4987 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_OIP6_DSCP, 4988 (1UL << 11) | (ltdefs->rx_oip6.lid << 8) | 4989 (ltdefs->rx_oip6.ltype_match << 4) | 4990 ltdefs->rx_oip6.ltype_mask); 4991 rvu_write64(rvu, blkaddr, NIX_AF_RX_DEF_IIP6_DSCP, 4992 (1UL << 11) | (ltdefs->rx_iip6.lid << 8) | 4993 (ltdefs->rx_iip6.ltype_match << 4) | 4994 ltdefs->rx_iip6.ltype_mask); 4995 } 4996 4997 static int nix_init_policer_context(struct rvu *rvu, struct nix_hw *nix_hw, 4998 int layer, int prof_idx) 4999 { 5000 struct nix_cn10k_aq_enq_req aq_req; 5001 int rc; 5002 5003 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5004 5005 aq_req.qidx = (prof_idx & 0x3FFF) | (layer << 14); 5006 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 5007 aq_req.op = NIX_AQ_INSTOP_INIT; 5008 5009 /* Context is all zeros, submit to AQ */ 5010 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5011 (struct nix_aq_enq_req *)&aq_req, NULL); 5012 if (rc) 5013 dev_err(rvu->dev, "Failed to INIT bandwidth profile layer %d profile %d\n", 5014 layer, prof_idx); 5015 return rc; 5016 } 5017 5018 static int nix_setup_ipolicers(struct rvu *rvu, 5019 struct nix_hw *nix_hw, int blkaddr) 5020 { 5021 struct rvu_hwinfo *hw = rvu->hw; 5022 struct nix_ipolicer *ipolicer; 5023 int err, layer, prof_idx; 5024 u64 cfg; 5025 5026 cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST); 5027 if (!(cfg & BIT_ULL(61))) { 5028 hw->cap.ipolicer = false; 5029 return 0; 5030 } 5031 5032 hw->cap.ipolicer = true; 5033 nix_hw->ipolicer = devm_kcalloc(rvu->dev, BAND_PROF_NUM_LAYERS, 5034 sizeof(*ipolicer), GFP_KERNEL); 5035 if (!nix_hw->ipolicer) 5036 return -ENOMEM; 5037 5038 cfg = rvu_read64(rvu, blkaddr, NIX_AF_PL_CONST); 5039 5040 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5041 ipolicer = &nix_hw->ipolicer[layer]; 5042 switch (layer) { 5043 case BAND_PROF_LEAF_LAYER: 5044 ipolicer->band_prof.max = cfg & 0XFFFF; 5045 break; 5046 case BAND_PROF_MID_LAYER: 5047 ipolicer->band_prof.max = (cfg >> 16) & 0XFFFF; 5048 break; 5049 case BAND_PROF_TOP_LAYER: 5050 ipolicer->band_prof.max = (cfg >> 32) & 0XFFFF; 5051 break; 5052 } 5053 5054 if (!ipolicer->band_prof.max) 5055 continue; 5056 5057 err = rvu_alloc_bitmap(&ipolicer->band_prof); 5058 if (err) 5059 return err; 5060 5061 ipolicer->pfvf_map = devm_kcalloc(rvu->dev, 5062 ipolicer->band_prof.max, 5063 sizeof(u16), GFP_KERNEL); 5064 if (!ipolicer->pfvf_map) 5065 return -ENOMEM; 5066 5067 ipolicer->match_id = devm_kcalloc(rvu->dev, 5068 ipolicer->band_prof.max, 5069 sizeof(u16), GFP_KERNEL); 5070 if (!ipolicer->match_id) 5071 return -ENOMEM; 5072 5073 for (prof_idx = 0; 5074 prof_idx < ipolicer->band_prof.max; prof_idx++) { 5075 /* Set AF as current owner for INIT ops to succeed */ 5076 ipolicer->pfvf_map[prof_idx] = 0x00; 5077 5078 /* There is no enable bit in the profile context, 5079 * so no context disable. So let's INIT them here 5080 * so that PF/VF later on have to just do WRITE to 5081 * setup policer rates and config. 5082 */ 5083 err = nix_init_policer_context(rvu, nix_hw, 5084 layer, prof_idx); 5085 if (err) 5086 return err; 5087 } 5088 5089 /* Allocate memory for maintaining ref_counts for MID level 5090 * profiles, this will be needed for leaf layer profiles' 5091 * aggregation. 5092 */ 5093 if (layer != BAND_PROF_MID_LAYER) 5094 continue; 5095 5096 ipolicer->ref_count = devm_kcalloc(rvu->dev, 5097 ipolicer->band_prof.max, 5098 sizeof(u16), GFP_KERNEL); 5099 if (!ipolicer->ref_count) 5100 return -ENOMEM; 5101 } 5102 5103 /* Set policer timeunit to 2us ie (19 + 1) * 100 nsec = 2us */ 5104 rvu_write64(rvu, blkaddr, NIX_AF_PL_TS, 19); 5105 5106 nix_config_rx_pkt_policer_precolor(rvu, blkaddr); 5107 5108 return 0; 5109 } 5110 5111 static void nix_ipolicer_freemem(struct rvu *rvu, struct nix_hw *nix_hw) 5112 { 5113 struct nix_ipolicer *ipolicer; 5114 int layer; 5115 5116 if (!rvu->hw->cap.ipolicer) 5117 return; 5118 5119 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5120 ipolicer = &nix_hw->ipolicer[layer]; 5121 5122 if (!ipolicer->band_prof.max) 5123 continue; 5124 5125 kfree(ipolicer->band_prof.bmap); 5126 } 5127 } 5128 5129 static int nix_verify_bandprof(struct nix_cn10k_aq_enq_req *req, 5130 struct nix_hw *nix_hw, u16 pcifunc) 5131 { 5132 struct nix_ipolicer *ipolicer; 5133 int layer, hi_layer, prof_idx; 5134 5135 /* Bits [15:14] in profile index represent layer */ 5136 layer = (req->qidx >> 14) & 0x03; 5137 prof_idx = req->qidx & 0x3FFF; 5138 5139 ipolicer = &nix_hw->ipolicer[layer]; 5140 if (prof_idx >= ipolicer->band_prof.max) 5141 return -EINVAL; 5142 5143 /* Check if the profile is allocated to the requesting PCIFUNC or not 5144 * with the exception of AF. AF is allowed to read and update contexts. 5145 */ 5146 if (pcifunc && ipolicer->pfvf_map[prof_idx] != pcifunc) 5147 return -EINVAL; 5148 5149 /* If this profile is linked to higher layer profile then check 5150 * if that profile is also allocated to the requesting PCIFUNC 5151 * or not. 5152 */ 5153 if (!req->prof.hl_en) 5154 return 0; 5155 5156 /* Leaf layer profile can link only to mid layer and 5157 * mid layer to top layer. 5158 */ 5159 if (layer == BAND_PROF_LEAF_LAYER) 5160 hi_layer = BAND_PROF_MID_LAYER; 5161 else if (layer == BAND_PROF_MID_LAYER) 5162 hi_layer = BAND_PROF_TOP_LAYER; 5163 else 5164 return -EINVAL; 5165 5166 ipolicer = &nix_hw->ipolicer[hi_layer]; 5167 prof_idx = req->prof.band_prof_id; 5168 if (prof_idx >= ipolicer->band_prof.max || 5169 ipolicer->pfvf_map[prof_idx] != pcifunc) 5170 return -EINVAL; 5171 5172 return 0; 5173 } 5174 5175 int rvu_mbox_handler_nix_bandprof_alloc(struct rvu *rvu, 5176 struct nix_bandprof_alloc_req *req, 5177 struct nix_bandprof_alloc_rsp *rsp) 5178 { 5179 int blkaddr, layer, prof, idx, err; 5180 u16 pcifunc = req->hdr.pcifunc; 5181 struct nix_ipolicer *ipolicer; 5182 struct nix_hw *nix_hw; 5183 5184 if (!rvu->hw->cap.ipolicer) 5185 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5186 5187 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5188 if (err) 5189 return err; 5190 5191 mutex_lock(&rvu->rsrc_lock); 5192 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5193 if (layer == BAND_PROF_INVAL_LAYER) 5194 continue; 5195 if (!req->prof_count[layer]) 5196 continue; 5197 5198 ipolicer = &nix_hw->ipolicer[layer]; 5199 for (idx = 0; idx < req->prof_count[layer]; idx++) { 5200 /* Allocate a max of 'MAX_BANDPROF_PER_PFFUNC' profiles */ 5201 if (idx == MAX_BANDPROF_PER_PFFUNC) 5202 break; 5203 5204 prof = rvu_alloc_rsrc(&ipolicer->band_prof); 5205 if (prof < 0) 5206 break; 5207 rsp->prof_count[layer]++; 5208 rsp->prof_idx[layer][idx] = prof; 5209 ipolicer->pfvf_map[prof] = pcifunc; 5210 } 5211 } 5212 mutex_unlock(&rvu->rsrc_lock); 5213 return 0; 5214 } 5215 5216 static int nix_free_all_bandprof(struct rvu *rvu, u16 pcifunc) 5217 { 5218 int blkaddr, layer, prof_idx, err; 5219 struct nix_ipolicer *ipolicer; 5220 struct nix_hw *nix_hw; 5221 5222 if (!rvu->hw->cap.ipolicer) 5223 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5224 5225 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5226 if (err) 5227 return err; 5228 5229 mutex_lock(&rvu->rsrc_lock); 5230 /* Free all the profiles allocated to the PCIFUNC */ 5231 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5232 if (layer == BAND_PROF_INVAL_LAYER) 5233 continue; 5234 ipolicer = &nix_hw->ipolicer[layer]; 5235 5236 for (prof_idx = 0; prof_idx < ipolicer->band_prof.max; prof_idx++) { 5237 if (ipolicer->pfvf_map[prof_idx] != pcifunc) 5238 continue; 5239 5240 /* Clear ratelimit aggregation, if any */ 5241 if (layer == BAND_PROF_LEAF_LAYER && 5242 ipolicer->match_id[prof_idx]) 5243 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 5244 5245 ipolicer->pfvf_map[prof_idx] = 0x00; 5246 ipolicer->match_id[prof_idx] = 0; 5247 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 5248 } 5249 } 5250 mutex_unlock(&rvu->rsrc_lock); 5251 return 0; 5252 } 5253 5254 int rvu_mbox_handler_nix_bandprof_free(struct rvu *rvu, 5255 struct nix_bandprof_free_req *req, 5256 struct msg_rsp *rsp) 5257 { 5258 int blkaddr, layer, prof_idx, idx, err; 5259 u16 pcifunc = req->hdr.pcifunc; 5260 struct nix_ipolicer *ipolicer; 5261 struct nix_hw *nix_hw; 5262 5263 if (req->free_all) 5264 return nix_free_all_bandprof(rvu, pcifunc); 5265 5266 if (!rvu->hw->cap.ipolicer) 5267 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5268 5269 err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5270 if (err) 5271 return err; 5272 5273 mutex_lock(&rvu->rsrc_lock); 5274 /* Free the requested profile indices */ 5275 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5276 if (layer == BAND_PROF_INVAL_LAYER) 5277 continue; 5278 if (!req->prof_count[layer]) 5279 continue; 5280 5281 ipolicer = &nix_hw->ipolicer[layer]; 5282 for (idx = 0; idx < req->prof_count[layer]; idx++) { 5283 prof_idx = req->prof_idx[layer][idx]; 5284 if (prof_idx >= ipolicer->band_prof.max || 5285 ipolicer->pfvf_map[prof_idx] != pcifunc) 5286 continue; 5287 5288 /* Clear ratelimit aggregation, if any */ 5289 if (layer == BAND_PROF_LEAF_LAYER && 5290 ipolicer->match_id[prof_idx]) 5291 nix_clear_ratelimit_aggr(rvu, nix_hw, prof_idx); 5292 5293 ipolicer->pfvf_map[prof_idx] = 0x00; 5294 ipolicer->match_id[prof_idx] = 0; 5295 rvu_free_rsrc(&ipolicer->band_prof, prof_idx); 5296 if (idx == MAX_BANDPROF_PER_PFFUNC) 5297 break; 5298 } 5299 } 5300 mutex_unlock(&rvu->rsrc_lock); 5301 return 0; 5302 } 5303 5304 int nix_aq_context_read(struct rvu *rvu, struct nix_hw *nix_hw, 5305 struct nix_cn10k_aq_enq_req *aq_req, 5306 struct nix_cn10k_aq_enq_rsp *aq_rsp, 5307 u16 pcifunc, u8 ctype, u32 qidx) 5308 { 5309 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5310 aq_req->hdr.pcifunc = pcifunc; 5311 aq_req->ctype = ctype; 5312 aq_req->op = NIX_AQ_INSTOP_READ; 5313 aq_req->qidx = qidx; 5314 5315 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5316 (struct nix_aq_enq_req *)aq_req, 5317 (struct nix_aq_enq_rsp *)aq_rsp); 5318 } 5319 5320 static int nix_ipolicer_map_leaf_midprofs(struct rvu *rvu, 5321 struct nix_hw *nix_hw, 5322 struct nix_cn10k_aq_enq_req *aq_req, 5323 struct nix_cn10k_aq_enq_rsp *aq_rsp, 5324 u32 leaf_prof, u16 mid_prof) 5325 { 5326 memset(aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5327 aq_req->hdr.pcifunc = 0x00; 5328 aq_req->ctype = NIX_AQ_CTYPE_BANDPROF; 5329 aq_req->op = NIX_AQ_INSTOP_WRITE; 5330 aq_req->qidx = leaf_prof; 5331 5332 aq_req->prof.band_prof_id = mid_prof; 5333 aq_req->prof_mask.band_prof_id = GENMASK(6, 0); 5334 aq_req->prof.hl_en = 1; 5335 aq_req->prof_mask.hl_en = 1; 5336 5337 return rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5338 (struct nix_aq_enq_req *)aq_req, 5339 (struct nix_aq_enq_rsp *)aq_rsp); 5340 } 5341 5342 int rvu_nix_setup_ratelimit_aggr(struct rvu *rvu, u16 pcifunc, 5343 u16 rq_idx, u16 match_id) 5344 { 5345 int leaf_prof, mid_prof, leaf_match; 5346 struct nix_cn10k_aq_enq_req aq_req; 5347 struct nix_cn10k_aq_enq_rsp aq_rsp; 5348 struct nix_ipolicer *ipolicer; 5349 struct nix_hw *nix_hw; 5350 int blkaddr, idx, rc; 5351 5352 if (!rvu->hw->cap.ipolicer) 5353 return 0; 5354 5355 rc = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr); 5356 if (rc) 5357 return rc; 5358 5359 /* Fetch the RQ's context to see if policing is enabled */ 5360 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, pcifunc, 5361 NIX_AQ_CTYPE_RQ, rq_idx); 5362 if (rc) { 5363 dev_err(rvu->dev, 5364 "%s: Failed to fetch RQ%d context of PFFUNC 0x%x\n", 5365 __func__, rq_idx, pcifunc); 5366 return rc; 5367 } 5368 5369 if (!aq_rsp.rq.policer_ena) 5370 return 0; 5371 5372 /* Get the bandwidth profile ID mapped to this RQ */ 5373 leaf_prof = aq_rsp.rq.band_prof_id; 5374 5375 ipolicer = &nix_hw->ipolicer[BAND_PROF_LEAF_LAYER]; 5376 ipolicer->match_id[leaf_prof] = match_id; 5377 5378 /* Check if any other leaf profile is marked with same match_id */ 5379 for (idx = 0; idx < ipolicer->band_prof.max; idx++) { 5380 if (idx == leaf_prof) 5381 continue; 5382 if (ipolicer->match_id[idx] != match_id) 5383 continue; 5384 5385 leaf_match = idx; 5386 break; 5387 } 5388 5389 if (idx == ipolicer->band_prof.max) 5390 return 0; 5391 5392 /* Fetch the matching profile's context to check if it's already 5393 * mapped to a mid level profile. 5394 */ 5395 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 5396 NIX_AQ_CTYPE_BANDPROF, leaf_match); 5397 if (rc) { 5398 dev_err(rvu->dev, 5399 "%s: Failed to fetch context of leaf profile %d\n", 5400 __func__, leaf_match); 5401 return rc; 5402 } 5403 5404 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 5405 if (aq_rsp.prof.hl_en) { 5406 /* Get Mid layer prof index and map leaf_prof index 5407 * also such that flows that are being steered 5408 * to different RQs and marked with same match_id 5409 * are rate limited in a aggregate fashion 5410 */ 5411 mid_prof = aq_rsp.prof.band_prof_id; 5412 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 5413 &aq_req, &aq_rsp, 5414 leaf_prof, mid_prof); 5415 if (rc) { 5416 dev_err(rvu->dev, 5417 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 5418 __func__, leaf_prof, mid_prof); 5419 goto exit; 5420 } 5421 5422 mutex_lock(&rvu->rsrc_lock); 5423 ipolicer->ref_count[mid_prof]++; 5424 mutex_unlock(&rvu->rsrc_lock); 5425 goto exit; 5426 } 5427 5428 /* Allocate a mid layer profile and 5429 * map both 'leaf_prof' and 'leaf_match' profiles to it. 5430 */ 5431 mutex_lock(&rvu->rsrc_lock); 5432 mid_prof = rvu_alloc_rsrc(&ipolicer->band_prof); 5433 if (mid_prof < 0) { 5434 dev_err(rvu->dev, 5435 "%s: Unable to allocate mid layer profile\n", __func__); 5436 mutex_unlock(&rvu->rsrc_lock); 5437 goto exit; 5438 } 5439 mutex_unlock(&rvu->rsrc_lock); 5440 ipolicer->pfvf_map[mid_prof] = 0x00; 5441 ipolicer->ref_count[mid_prof] = 0; 5442 5443 /* Initialize mid layer profile same as 'leaf_prof' */ 5444 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 5445 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 5446 if (rc) { 5447 dev_err(rvu->dev, 5448 "%s: Failed to fetch context of leaf profile %d\n", 5449 __func__, leaf_prof); 5450 goto exit; 5451 } 5452 5453 memset(&aq_req, 0, sizeof(struct nix_cn10k_aq_enq_req)); 5454 aq_req.hdr.pcifunc = 0x00; 5455 aq_req.qidx = (mid_prof & 0x3FFF) | (BAND_PROF_MID_LAYER << 14); 5456 aq_req.ctype = NIX_AQ_CTYPE_BANDPROF; 5457 aq_req.op = NIX_AQ_INSTOP_WRITE; 5458 memcpy(&aq_req.prof, &aq_rsp.prof, sizeof(struct nix_bandprof_s)); 5459 memset((char *)&aq_req.prof_mask, 0xff, sizeof(struct nix_bandprof_s)); 5460 /* Clear higher layer enable bit in the mid profile, just in case */ 5461 aq_req.prof.hl_en = 0; 5462 aq_req.prof_mask.hl_en = 1; 5463 5464 rc = rvu_nix_blk_aq_enq_inst(rvu, nix_hw, 5465 (struct nix_aq_enq_req *)&aq_req, NULL); 5466 if (rc) { 5467 dev_err(rvu->dev, 5468 "%s: Failed to INIT context of mid layer profile %d\n", 5469 __func__, mid_prof); 5470 goto exit; 5471 } 5472 5473 /* Map both leaf profiles to this mid layer profile */ 5474 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 5475 &aq_req, &aq_rsp, 5476 leaf_prof, mid_prof); 5477 if (rc) { 5478 dev_err(rvu->dev, 5479 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 5480 __func__, leaf_prof, mid_prof); 5481 goto exit; 5482 } 5483 5484 mutex_lock(&rvu->rsrc_lock); 5485 ipolicer->ref_count[mid_prof]++; 5486 mutex_unlock(&rvu->rsrc_lock); 5487 5488 rc = nix_ipolicer_map_leaf_midprofs(rvu, nix_hw, 5489 &aq_req, &aq_rsp, 5490 leaf_match, mid_prof); 5491 if (rc) { 5492 dev_err(rvu->dev, 5493 "%s: Failed to map leaf(%d) and mid(%d) profiles\n", 5494 __func__, leaf_match, mid_prof); 5495 ipolicer->ref_count[mid_prof]--; 5496 goto exit; 5497 } 5498 5499 mutex_lock(&rvu->rsrc_lock); 5500 ipolicer->ref_count[mid_prof]++; 5501 mutex_unlock(&rvu->rsrc_lock); 5502 5503 exit: 5504 return rc; 5505 } 5506 5507 /* Called with mutex rsrc_lock */ 5508 static void nix_clear_ratelimit_aggr(struct rvu *rvu, struct nix_hw *nix_hw, 5509 u32 leaf_prof) 5510 { 5511 struct nix_cn10k_aq_enq_req aq_req; 5512 struct nix_cn10k_aq_enq_rsp aq_rsp; 5513 struct nix_ipolicer *ipolicer; 5514 u16 mid_prof; 5515 int rc; 5516 5517 mutex_unlock(&rvu->rsrc_lock); 5518 5519 rc = nix_aq_context_read(rvu, nix_hw, &aq_req, &aq_rsp, 0x00, 5520 NIX_AQ_CTYPE_BANDPROF, leaf_prof); 5521 5522 mutex_lock(&rvu->rsrc_lock); 5523 if (rc) { 5524 dev_err(rvu->dev, 5525 "%s: Failed to fetch context of leaf profile %d\n", 5526 __func__, leaf_prof); 5527 return; 5528 } 5529 5530 if (!aq_rsp.prof.hl_en) 5531 return; 5532 5533 mid_prof = aq_rsp.prof.band_prof_id; 5534 ipolicer = &nix_hw->ipolicer[BAND_PROF_MID_LAYER]; 5535 ipolicer->ref_count[mid_prof]--; 5536 /* If ref_count is zero, free mid layer profile */ 5537 if (!ipolicer->ref_count[mid_prof]) { 5538 ipolicer->pfvf_map[mid_prof] = 0x00; 5539 rvu_free_rsrc(&ipolicer->band_prof, mid_prof); 5540 } 5541 } 5542 5543 int rvu_mbox_handler_nix_bandprof_get_hwinfo(struct rvu *rvu, struct msg_req *req, 5544 struct nix_bandprof_get_hwinfo_rsp *rsp) 5545 { 5546 struct nix_ipolicer *ipolicer; 5547 int blkaddr, layer, err; 5548 struct nix_hw *nix_hw; 5549 u64 tu; 5550 5551 if (!rvu->hw->cap.ipolicer) 5552 return NIX_AF_ERR_IPOLICER_NOTSUPP; 5553 5554 err = nix_get_struct_ptrs(rvu, req->hdr.pcifunc, &nix_hw, &blkaddr); 5555 if (err) 5556 return err; 5557 5558 /* Return number of bandwidth profiles free at each layer */ 5559 mutex_lock(&rvu->rsrc_lock); 5560 for (layer = 0; layer < BAND_PROF_NUM_LAYERS; layer++) { 5561 if (layer == BAND_PROF_INVAL_LAYER) 5562 continue; 5563 5564 ipolicer = &nix_hw->ipolicer[layer]; 5565 rsp->prof_count[layer] = rvu_rsrc_free_count(&ipolicer->band_prof); 5566 } 5567 mutex_unlock(&rvu->rsrc_lock); 5568 5569 /* Set the policer timeunit in nanosec */ 5570 tu = rvu_read64(rvu, blkaddr, NIX_AF_PL_TS) & GENMASK_ULL(9, 0); 5571 rsp->policer_timeunit = (tu + 1) * 100; 5572 5573 return 0; 5574 } 5575