1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2020 Marvell. 5 * 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/pci.h> 10 #include "rvu_struct.h" 11 #include "rvu_reg.h" 12 #include "mbox.h" 13 #include "rvu.h" 14 15 /* CPT PF device id */ 16 #define PCI_DEVID_OTX2_CPT_PF 0xA0FD 17 #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2 18 19 /* Length of initial context fetch in 128 byte words */ 20 #define CPT_CTX_ILEN 2 21 22 #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \ 23 ({ \ 24 u64 free_sts = 0, busy_sts = 0; \ 25 typeof(rsp) _rsp = rsp; \ 26 u32 e, i; \ 27 \ 28 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \ 29 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \ 30 if (reg & 0x1) \ 31 busy_sts |= 1ULL << i; \ 32 \ 33 if (reg & 0x2) \ 34 free_sts |= 1ULL << i; \ 35 } \ 36 (_rsp)->busy_sts_##etype = busy_sts; \ 37 (_rsp)->free_sts_##etype = free_sts; \ 38 }) 39 40 static irqreturn_t rvu_cpt_af_flt_intr_handler(int irq, void *ptr) 41 { 42 struct rvu_block *block = ptr; 43 struct rvu *rvu = block->rvu; 44 int blkaddr = block->addr; 45 u64 reg0, reg1, reg2; 46 47 reg0 = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(0)); 48 reg1 = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(1)); 49 if (!is_rvu_otx2(rvu)) { 50 reg2 = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(2)); 51 dev_err_ratelimited(rvu->dev, 52 "Received CPTAF FLT irq : 0x%llx, 0x%llx, 0x%llx", 53 reg0, reg1, reg2); 54 } else { 55 dev_err_ratelimited(rvu->dev, 56 "Received CPTAF FLT irq : 0x%llx, 0x%llx", 57 reg0, reg1); 58 } 59 60 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(0), reg0); 61 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(1), reg1); 62 if (!is_rvu_otx2(rvu)) 63 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(2), reg2); 64 65 return IRQ_HANDLED; 66 } 67 68 static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr) 69 { 70 struct rvu_block *block = ptr; 71 struct rvu *rvu = block->rvu; 72 int blkaddr = block->addr; 73 u64 reg; 74 75 reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT); 76 dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg); 77 78 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg); 79 return IRQ_HANDLED; 80 } 81 82 static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr) 83 { 84 struct rvu_block *block = ptr; 85 struct rvu *rvu = block->rvu; 86 int blkaddr = block->addr; 87 u64 reg; 88 89 reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT); 90 dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg); 91 92 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg); 93 return IRQ_HANDLED; 94 } 95 96 static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs, 97 irq_handler_t handler, 98 const char *name) 99 { 100 struct rvu *rvu = block->rvu; 101 int ret; 102 103 ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0, 104 name, block); 105 if (ret) { 106 dev_err(rvu->dev, "RVUAF: %s irq registration failed", name); 107 return ret; 108 } 109 110 WARN_ON(rvu->irq_allocated[irq_offs]); 111 rvu->irq_allocated[irq_offs] = true; 112 return 0; 113 } 114 115 static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off) 116 { 117 struct rvu *rvu = block->rvu; 118 int blkaddr = block->addr; 119 int i; 120 121 /* Disable all CPT AF interrupts */ 122 for (i = 0; i < CPT_10K_AF_INT_VEC_RVU; i++) 123 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), 0x1); 124 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1); 125 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1); 126 127 for (i = 0; i < CPT_10K_AF_INT_VEC_CNT; i++) 128 if (rvu->irq_allocated[off + i]) { 129 free_irq(pci_irq_vector(rvu->pdev, off + i), block); 130 rvu->irq_allocated[off + i] = false; 131 } 132 } 133 134 static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr) 135 { 136 struct rvu_hwinfo *hw = rvu->hw; 137 struct rvu_block *block; 138 int i, offs; 139 140 if (!is_block_implemented(rvu->hw, blkaddr)) 141 return; 142 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; 143 if (!offs) { 144 dev_warn(rvu->dev, 145 "Failed to get CPT_AF_INT vector offsets\n"); 146 return; 147 } 148 block = &hw->block[blkaddr]; 149 if (!is_rvu_otx2(rvu)) 150 return cpt_10k_unregister_interrupts(block, offs); 151 152 /* Disable all CPT AF interrupts */ 153 for (i = 0; i < CPT_AF_INT_VEC_RVU; i++) 154 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), 0x1); 155 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1); 156 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1); 157 158 for (i = 0; i < CPT_AF_INT_VEC_CNT; i++) 159 if (rvu->irq_allocated[offs + i]) { 160 free_irq(pci_irq_vector(rvu->pdev, offs + i), block); 161 rvu->irq_allocated[offs + i] = false; 162 } 163 } 164 165 void rvu_cpt_unregister_interrupts(struct rvu *rvu) 166 { 167 cpt_unregister_interrupts(rvu, BLKADDR_CPT0); 168 cpt_unregister_interrupts(rvu, BLKADDR_CPT1); 169 } 170 171 static int cpt_10k_register_interrupts(struct rvu_block *block, int off) 172 { 173 struct rvu *rvu = block->rvu; 174 int blkaddr = block->addr; 175 char irq_name[16]; 176 int i, ret; 177 178 for (i = CPT_10K_AF_INT_VEC_FLT0; i < CPT_10K_AF_INT_VEC_RVU; i++) { 179 snprintf(irq_name, sizeof(irq_name), "CPTAF FLT%d", i); 180 ret = rvu_cpt_do_register_interrupt(block, off + i, 181 rvu_cpt_af_flt_intr_handler, 182 irq_name); 183 if (ret) 184 goto err; 185 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 0x1); 186 } 187 188 ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RVU, 189 rvu_cpt_af_rvu_intr_handler, 190 "CPTAF RVU"); 191 if (ret) 192 goto err; 193 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1); 194 195 ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RAS, 196 rvu_cpt_af_ras_intr_handler, 197 "CPTAF RAS"); 198 if (ret) 199 goto err; 200 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1); 201 202 return 0; 203 err: 204 rvu_cpt_unregister_interrupts(rvu); 205 return ret; 206 } 207 208 static int cpt_register_interrupts(struct rvu *rvu, int blkaddr) 209 { 210 struct rvu_hwinfo *hw = rvu->hw; 211 struct rvu_block *block; 212 int i, offs, ret = 0; 213 char irq_name[16]; 214 215 if (!is_block_implemented(rvu->hw, blkaddr)) 216 return 0; 217 218 block = &hw->block[blkaddr]; 219 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF; 220 if (!offs) { 221 dev_warn(rvu->dev, 222 "Failed to get CPT_AF_INT vector offsets\n"); 223 return 0; 224 } 225 226 if (!is_rvu_otx2(rvu)) 227 return cpt_10k_register_interrupts(block, offs); 228 229 for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) { 230 snprintf(irq_name, sizeof(irq_name), "CPTAF FLT%d", i); 231 ret = rvu_cpt_do_register_interrupt(block, offs + i, 232 rvu_cpt_af_flt_intr_handler, 233 irq_name); 234 if (ret) 235 goto err; 236 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 0x1); 237 } 238 239 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU, 240 rvu_cpt_af_rvu_intr_handler, 241 "CPTAF RVU"); 242 if (ret) 243 goto err; 244 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1); 245 246 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS, 247 rvu_cpt_af_ras_intr_handler, 248 "CPTAF RAS"); 249 if (ret) 250 goto err; 251 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1); 252 253 return 0; 254 err: 255 rvu_cpt_unregister_interrupts(rvu); 256 return ret; 257 } 258 259 int rvu_cpt_register_interrupts(struct rvu *rvu) 260 { 261 int ret; 262 263 ret = cpt_register_interrupts(rvu, BLKADDR_CPT0); 264 if (ret) 265 return ret; 266 267 return cpt_register_interrupts(rvu, BLKADDR_CPT1); 268 } 269 270 static int get_cpt_pf_num(struct rvu *rvu) 271 { 272 int i, domain_nr, cpt_pf_num = -1; 273 struct pci_dev *pdev; 274 275 domain_nr = pci_domain_nr(rvu->pdev->bus); 276 for (i = 0; i < rvu->hw->total_pfs; i++) { 277 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0); 278 if (!pdev) 279 continue; 280 281 if (pdev->device == PCI_DEVID_OTX2_CPT_PF || 282 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) { 283 cpt_pf_num = i; 284 put_device(&pdev->dev); 285 break; 286 } 287 put_device(&pdev->dev); 288 } 289 return cpt_pf_num; 290 } 291 292 static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc) 293 { 294 int cpt_pf_num = get_cpt_pf_num(rvu); 295 296 if (rvu_get_pf(pcifunc) != cpt_pf_num) 297 return false; 298 if (pcifunc & RVU_PFVF_FUNC_MASK) 299 return false; 300 301 return true; 302 } 303 304 static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc) 305 { 306 int cpt_pf_num = get_cpt_pf_num(rvu); 307 308 if (rvu_get_pf(pcifunc) != cpt_pf_num) 309 return false; 310 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 311 return false; 312 313 return true; 314 } 315 316 static int validate_and_get_cpt_blkaddr(int req_blkaddr) 317 { 318 int blkaddr; 319 320 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0; 321 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1) 322 return -EINVAL; 323 324 return blkaddr; 325 } 326 327 int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu, 328 struct cpt_lf_alloc_req_msg *req, 329 struct msg_rsp *rsp) 330 { 331 u16 pcifunc = req->hdr.pcifunc; 332 struct rvu_block *block; 333 int cptlf, blkaddr; 334 int num_lfs, slot; 335 u64 val; 336 337 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 338 if (blkaddr < 0) 339 return blkaddr; 340 341 if (req->eng_grpmsk == 0x0) 342 return CPT_AF_ERR_GRP_INVALID; 343 344 block = &rvu->hw->block[blkaddr]; 345 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 346 block->addr); 347 if (!num_lfs) 348 return CPT_AF_ERR_LF_INVALID; 349 350 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ 351 if (req->nix_pf_func) { 352 /* If default, use 'this' CPTLF's PFFUNC */ 353 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC) 354 req->nix_pf_func = pcifunc; 355 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX)) 356 return CPT_AF_ERR_NIX_PF_FUNC_INVALID; 357 } 358 359 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ 360 if (req->sso_pf_func) { 361 /* If default, use 'this' CPTLF's PFFUNC */ 362 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC) 363 req->sso_pf_func = pcifunc; 364 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO)) 365 return CPT_AF_ERR_SSO_PF_FUNC_INVALID; 366 } 367 368 for (slot = 0; slot < num_lfs; slot++) { 369 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 370 if (cptlf < 0) 371 return CPT_AF_ERR_LF_INVALID; 372 373 /* Set CPT LF group and priority */ 374 val = (u64)req->eng_grpmsk << 48 | 1; 375 if (!is_rvu_otx2(rvu)) 376 val |= (CPT_CTX_ILEN << 17); 377 378 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 379 380 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set 381 * on reset. 382 */ 383 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 384 val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32)); 385 val |= ((u64)req->nix_pf_func << 48 | 386 (u64)req->sso_pf_func << 32); 387 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 388 } 389 390 return 0; 391 } 392 393 static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr) 394 { 395 u16 pcifunc = req->hdr.pcifunc; 396 int num_lfs, cptlf, slot, err; 397 struct rvu_block *block; 398 399 block = &rvu->hw->block[blkaddr]; 400 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 401 block->addr); 402 if (!num_lfs) 403 return 0; 404 405 for (slot = 0; slot < num_lfs; slot++) { 406 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 407 if (cptlf < 0) 408 return CPT_AF_ERR_LF_INVALID; 409 410 /* Perform teardown */ 411 rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot); 412 413 /* Reset LF */ 414 err = rvu_lf_reset(rvu, block, cptlf); 415 if (err) { 416 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n", 417 block->addr, cptlf); 418 } 419 } 420 421 return 0; 422 } 423 424 int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req, 425 struct msg_rsp *rsp) 426 { 427 int ret; 428 429 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0); 430 if (ret) 431 return ret; 432 433 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 434 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1); 435 436 return ret; 437 } 438 439 static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf, 440 struct cpt_inline_ipsec_cfg_msg *req) 441 { 442 u16 sso_pf_func = req->sso_pf_func; 443 u8 nix_sel; 444 u64 val; 445 446 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 447 if (req->enable && (val & BIT_ULL(16))) { 448 /* IPSec inline outbound path is already enabled for a given 449 * CPT LF, HRM states that inline inbound & outbound paths 450 * must not be enabled at the same time for a given CPT LF 451 */ 452 return CPT_AF_ERR_INLINE_IPSEC_INB_ENA; 453 } 454 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ 455 if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO)) 456 return CPT_AF_ERR_SSO_PF_FUNC_INVALID; 457 458 nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0; 459 /* Enable CPT LF for IPsec inline inbound operations */ 460 if (req->enable) 461 val |= BIT_ULL(9); 462 else 463 val &= ~BIT_ULL(9); 464 465 val |= (u64)nix_sel << 8; 466 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 467 468 if (sso_pf_func) { 469 /* Set SSO_PF_FUNC */ 470 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 471 val |= (u64)sso_pf_func << 32; 472 val |= (u64)req->nix_pf_func << 48; 473 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 474 } 475 if (req->sso_pf_func_ovrd) 476 /* Set SSO_PF_FUNC_OVRD for inline IPSec */ 477 rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1); 478 479 /* Configure the X2P Link register with the cpt base channel number and 480 * range of channels it should propagate to X2P 481 */ 482 if (!is_rvu_otx2(rvu)) { 483 val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16); 484 val |= rvu->hw->cpt_chan_base; 485 486 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val); 487 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val); 488 } 489 490 return 0; 491 } 492 493 static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf, 494 struct cpt_inline_ipsec_cfg_msg *req) 495 { 496 u16 nix_pf_func = req->nix_pf_func; 497 int nix_blkaddr; 498 u8 nix_sel; 499 u64 val; 500 501 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 502 if (req->enable && (val & BIT_ULL(9))) { 503 /* IPSec inline inbound path is already enabled for a given 504 * CPT LF, HRM states that inline inbound & outbound paths 505 * must not be enabled at the same time for a given CPT LF 506 */ 507 return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA; 508 } 509 510 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ 511 if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX)) 512 return CPT_AF_ERR_NIX_PF_FUNC_INVALID; 513 514 /* Enable CPT LF for IPsec inline outbound operations */ 515 if (req->enable) 516 val |= BIT_ULL(16); 517 else 518 val &= ~BIT_ULL(16); 519 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 520 521 if (nix_pf_func) { 522 /* Set NIX_PF_FUNC */ 523 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf)); 524 val |= (u64)nix_pf_func << 48; 525 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 526 527 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func); 528 nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1; 529 530 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf)); 531 val |= (u64)nix_sel << 8; 532 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 533 } 534 535 return 0; 536 } 537 538 int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu, 539 struct cpt_inline_ipsec_cfg_msg *req, 540 struct msg_rsp *rsp) 541 { 542 u16 pcifunc = req->hdr.pcifunc; 543 struct rvu_block *block; 544 int cptlf, blkaddr, ret; 545 u16 actual_slot; 546 547 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc, 548 req->slot, &actual_slot); 549 if (blkaddr < 0) 550 return CPT_AF_ERR_LF_INVALID; 551 552 block = &rvu->hw->block[blkaddr]; 553 554 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot); 555 if (cptlf < 0) 556 return CPT_AF_ERR_LF_INVALID; 557 558 switch (req->dir) { 559 case CPT_INLINE_INBOUND: 560 ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req); 561 break; 562 563 case CPT_INLINE_OUTBOUND: 564 ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req); 565 break; 566 567 default: 568 return CPT_AF_ERR_PARAM; 569 } 570 571 return ret; 572 } 573 574 static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req) 575 { 576 u64 offset = req->reg_offset; 577 int blkaddr, num_lfs, lf; 578 struct rvu_block *block; 579 struct rvu_pfvf *pfvf; 580 581 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 582 if (blkaddr < 0) 583 return blkaddr; 584 585 /* Registers that can be accessed from PF/VF */ 586 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) || 587 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) { 588 if (offset & 7) 589 return false; 590 591 lf = (offset & 0xFFF) >> 3; 592 block = &rvu->hw->block[blkaddr]; 593 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 594 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr); 595 if (lf >= num_lfs) 596 /* Slot is not valid for that PF/VF */ 597 return false; 598 599 /* Translate local LF used by VFs to global CPT LF */ 600 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr], 601 req->hdr.pcifunc, lf); 602 if (lf < 0) 603 return false; 604 605 return true; 606 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) { 607 /* Registers that can be accessed from PF */ 608 switch (offset) { 609 case CPT_AF_CTL: 610 case CPT_AF_PF_FUNC: 611 case CPT_AF_BLK_RST: 612 case CPT_AF_CONSTANTS1: 613 case CPT_AF_CTX_FLUSH_TIMER: 614 return true; 615 } 616 617 switch (offset & 0xFF000) { 618 case CPT_AF_EXEX_STS(0): 619 case CPT_AF_EXEX_CTL(0): 620 case CPT_AF_EXEX_CTL2(0): 621 case CPT_AF_EXEX_UCODE_BASE(0): 622 if (offset & 7) 623 return false; 624 break; 625 default: 626 return false; 627 } 628 return true; 629 } 630 return false; 631 } 632 633 int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu, 634 struct cpt_rd_wr_reg_msg *req, 635 struct cpt_rd_wr_reg_msg *rsp) 636 { 637 int blkaddr; 638 639 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 640 if (blkaddr < 0) 641 return blkaddr; 642 643 /* This message is accepted only if sent from CPT PF/VF */ 644 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 645 !is_cpt_vf(rvu, req->hdr.pcifunc)) 646 return CPT_AF_ERR_ACCESS_DENIED; 647 648 rsp->reg_offset = req->reg_offset; 649 rsp->ret_val = req->ret_val; 650 rsp->is_write = req->is_write; 651 652 if (!is_valid_offset(rvu, req)) 653 return CPT_AF_ERR_ACCESS_DENIED; 654 655 if (req->is_write) 656 rvu_write64(rvu, blkaddr, req->reg_offset, req->val); 657 else 658 rsp->val = rvu_read64(rvu, blkaddr, req->reg_offset); 659 660 return 0; 661 } 662 663 static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 664 { 665 if (is_rvu_otx2(rvu)) 666 return; 667 668 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC); 669 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC); 670 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC); 671 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr, 672 CPT_AF_CTX_AOP_LATENCY_PC); 673 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC); 674 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr, 675 CPT_AF_CTX_IFETCH_LATENCY_PC); 676 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 677 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr, 678 CPT_AF_CTX_FFETCH_LATENCY_PC); 679 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 680 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr, 681 CPT_AF_CTX_FFETCH_LATENCY_PC); 682 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 683 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr, 684 CPT_AF_CTX_FFETCH_LATENCY_PC); 685 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR); 686 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID); 687 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER); 688 689 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME); 690 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG); 691 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS); 692 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS); 693 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG); 694 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0)); 695 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1)); 696 } 697 698 static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 699 { 700 u16 max_ses, max_ies, max_aes; 701 u32 e_min = 0, e_max = 0; 702 u64 reg; 703 704 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1); 705 max_ses = reg & 0xffff; 706 max_ies = (reg >> 16) & 0xffff; 707 max_aes = (reg >> 32) & 0xffff; 708 709 /* Get AE status */ 710 e_min = max_ses + max_ies; 711 e_max = max_ses + max_ies + max_aes; 712 cpt_get_eng_sts(e_min, e_max, rsp, ae); 713 /* Get SE status */ 714 e_min = 0; 715 e_max = max_ses; 716 cpt_get_eng_sts(e_min, e_max, rsp, se); 717 /* Get IE status */ 718 e_min = max_ses; 719 e_max = max_ses + max_ies; 720 cpt_get_eng_sts(e_min, e_max, rsp, ie); 721 } 722 723 int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req, 724 struct cpt_sts_rsp *rsp) 725 { 726 int blkaddr; 727 728 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 729 if (blkaddr < 0) 730 return blkaddr; 731 732 /* This message is accepted only if sent from CPT PF/VF */ 733 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 734 !is_cpt_vf(rvu, req->hdr.pcifunc)) 735 return CPT_AF_ERR_ACCESS_DENIED; 736 737 get_ctx_pc(rvu, rsp, blkaddr); 738 739 /* Get CPT engines status */ 740 get_eng_sts(rvu, rsp, blkaddr); 741 742 /* Read CPT instruction PC registers */ 743 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC); 744 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC); 745 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC); 746 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC); 747 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC); 748 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr, 749 CPT_AF_ACTIVE_CYCLES_PC); 750 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO); 751 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT); 752 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG); 753 754 return 0; 755 } 756 757 #define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48) 758 #define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32) 759 #define RXC_ACTIVE_THRES GENMASK_ULL(27, 16) 760 #define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0) 761 #define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48) 762 #define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48) 763 764 static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req, 765 int blkaddr) 766 { 767 u64 dfrg_reg; 768 769 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres); 770 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit); 771 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres); 772 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit); 773 774 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step); 775 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg); 776 } 777 778 int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu, 779 struct cpt_rxc_time_cfg_req *req, 780 struct msg_rsp *rsp) 781 { 782 int blkaddr; 783 784 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 785 if (blkaddr < 0) 786 return blkaddr; 787 788 /* This message is accepted only if sent from CPT PF/VF */ 789 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 790 !is_cpt_vf(rvu, req->hdr.pcifunc)) 791 return CPT_AF_ERR_ACCESS_DENIED; 792 793 cpt_rxc_time_cfg(rvu, req, blkaddr); 794 795 return 0; 796 } 797 798 int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req, 799 struct msg_rsp *rsp) 800 { 801 return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc); 802 } 803 804 static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr) 805 { 806 struct cpt_rxc_time_cfg_req req; 807 int timeout = 2000; 808 u64 reg; 809 810 if (is_rvu_otx2(rvu)) 811 return; 812 813 /* Set time limit to minimum values, so that rxc entries will be 814 * flushed out quickly. 815 */ 816 req.step = 1; 817 req.zombie_thres = 1; 818 req.zombie_limit = 1; 819 req.active_thres = 1; 820 req.active_limit = 1; 821 822 cpt_rxc_time_cfg(rvu, &req, blkaddr); 823 824 do { 825 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS); 826 udelay(1); 827 if (FIELD_GET(RXC_ACTIVE_COUNT, reg)) 828 timeout--; 829 else 830 break; 831 } while (timeout); 832 833 if (timeout == 0) 834 dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n"); 835 836 timeout = 2000; 837 do { 838 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS); 839 udelay(1); 840 if (FIELD_GET(RXC_ZOMBIE_COUNT, reg)) 841 timeout--; 842 else 843 break; 844 } while (timeout); 845 846 if (timeout == 0) 847 dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n"); 848 } 849 850 #define INPROG_INFLIGHT(reg) ((reg) & 0x1FF) 851 #define INPROG_GRB_PARTIAL(reg) ((reg) & BIT_ULL(31)) 852 #define INPROG_GRB(reg) (((reg) >> 32) & 0xFF) 853 #define INPROG_GWB(reg) (((reg) >> 40) & 0xFF) 854 855 static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot) 856 { 857 int i = 0, hard_lp_ctr = 100000; 858 u64 inprog, grp_ptr; 859 u16 nq_ptr, dq_ptr; 860 861 /* Disable instructions enqueuing */ 862 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0); 863 864 /* Disable executions in the LF's queue */ 865 inprog = rvu_read64(rvu, blkaddr, 866 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 867 inprog &= ~BIT_ULL(16); 868 rvu_write64(rvu, blkaddr, 869 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog); 870 871 /* Wait for CPT queue to become execution-quiescent */ 872 do { 873 inprog = rvu_read64(rvu, blkaddr, 874 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 875 if (INPROG_GRB_PARTIAL(inprog)) { 876 i = 0; 877 hard_lp_ctr--; 878 } else { 879 i++; 880 } 881 882 grp_ptr = rvu_read64(rvu, blkaddr, 883 CPT_AF_BAR2_ALIASX(slot, 884 CPT_LF_Q_GRP_PTR)); 885 nq_ptr = (grp_ptr >> 32) & 0x7FFF; 886 dq_ptr = grp_ptr & 0x7FFF; 887 888 } while (hard_lp_ctr && (i < 10) && (nq_ptr != dq_ptr)); 889 890 if (hard_lp_ctr == 0) 891 dev_warn(rvu->dev, "CPT FLR hits hard loop counter\n"); 892 893 i = 0; 894 hard_lp_ctr = 100000; 895 do { 896 inprog = rvu_read64(rvu, blkaddr, 897 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 898 899 if ((INPROG_INFLIGHT(inprog) == 0) && 900 (INPROG_GWB(inprog) < 40) && 901 ((INPROG_GRB(inprog) == 0) || 902 (INPROG_GRB((inprog)) == 40))) { 903 i++; 904 } else { 905 i = 0; 906 hard_lp_ctr--; 907 } 908 } while (hard_lp_ctr && (i < 10)); 909 910 if (hard_lp_ctr == 0) 911 dev_warn(rvu->dev, "CPT FLR hits hard loop counter\n"); 912 } 913 914 int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot) 915 { 916 u64 reg; 917 918 if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc)) 919 cpt_rxc_teardown(rvu, blkaddr); 920 921 /* Enable BAR2 ALIAS for this pcifunc. */ 922 reg = BIT_ULL(16) | pcifunc; 923 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg); 924 925 cpt_lf_disable_iqueue(rvu, blkaddr, slot); 926 927 /* Set group drop to help clear out hardware */ 928 reg = rvu_read64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 929 reg |= BIT_ULL(17); 930 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), reg); 931 932 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0); 933 934 return 0; 935 } 936 937 #define CPT_RES_LEN 16 938 #define CPT_SE_IE_EGRP 1ULL 939 940 static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr, 941 int nix_blkaddr) 942 { 943 int cpt_pf_num = get_cpt_pf_num(rvu); 944 struct cpt_inst_lmtst_req *req; 945 dma_addr_t res_daddr; 946 int timeout = 3000; 947 u8 cpt_idx; 948 u64 *inst; 949 u16 *res; 950 int rc; 951 952 res = kzalloc(CPT_RES_LEN, GFP_KERNEL); 953 if (!res) 954 return -ENOMEM; 955 956 res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN, 957 DMA_BIDIRECTIONAL); 958 if (dma_mapping_error(rvu->dev, res_daddr)) { 959 dev_err(rvu->dev, "DMA mapping failed for CPT result\n"); 960 rc = -EFAULT; 961 goto res_free; 962 } 963 *res = 0xFFFF; 964 965 /* Send mbox message to CPT PF */ 966 req = (struct cpt_inst_lmtst_req *) 967 otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up, 968 cpt_pf_num, sizeof(*req), 969 sizeof(struct msg_rsp)); 970 if (!req) { 971 rc = -ENOMEM; 972 goto res_daddr_unmap; 973 } 974 req->hdr.sig = OTX2_MBOX_REQ_SIG; 975 req->hdr.id = MBOX_MSG_CPT_INST_LMTST; 976 977 inst = req->inst; 978 /* Prepare CPT_INST_S */ 979 inst[0] = 0; 980 inst[1] = res_daddr; 981 /* AF PF FUNC */ 982 inst[2] = 0; 983 /* Set QORD */ 984 inst[3] = 1; 985 inst[4] = 0; 986 inst[5] = 0; 987 inst[6] = 0; 988 /* Set EGRP */ 989 inst[7] = CPT_SE_IE_EGRP << 61; 990 991 /* Subtract 1 from the NIX-CPT credit count to preserve 992 * credit counts. 993 */ 994 cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1; 995 rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx), 996 BIT_ULL(22) - 1); 997 998 otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num); 999 rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num); 1000 if (rc) 1001 dev_warn(rvu->dev, "notification to pf %d failed\n", 1002 cpt_pf_num); 1003 /* Wait for CPT instruction to be completed */ 1004 do { 1005 mdelay(1); 1006 if (*res == 0xFFFF) 1007 timeout--; 1008 else 1009 break; 1010 } while (timeout); 1011 1012 if (timeout == 0) 1013 dev_warn(rvu->dev, "Poll for result hits hard loop counter\n"); 1014 1015 res_daddr_unmap: 1016 dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL); 1017 res_free: 1018 kfree(res); 1019 1020 return 0; 1021 } 1022 1023 #define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46) 1024 #define CTX_CAM_CPTR GENMASK_ULL(45, 0) 1025 1026 int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc) 1027 { 1028 int nix_blkaddr, blkaddr; 1029 u16 max_ctx_entries, i; 1030 int slot = 0, num_lfs; 1031 u64 reg, cam_data; 1032 int rc; 1033 1034 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc); 1035 if (nix_blkaddr < 0) 1036 return -EINVAL; 1037 1038 if (is_rvu_otx2(rvu)) 1039 return 0; 1040 1041 blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0; 1042 1043 /* Submit CPT_INST_S to track when all packets have been 1044 * flushed through for the NIX PF FUNC in inline inbound case. 1045 */ 1046 rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr); 1047 if (rc) 1048 return rc; 1049 1050 /* Wait for rxc entries to be flushed out */ 1051 cpt_rxc_teardown(rvu, blkaddr); 1052 1053 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0); 1054 max_ctx_entries = (reg >> 48) & 0xFFF; 1055 1056 mutex_lock(&rvu->rsrc_lock); 1057 1058 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 1059 blkaddr); 1060 if (num_lfs == 0) { 1061 dev_warn(rvu->dev, "CPT LF is not configured\n"); 1062 goto unlock; 1063 } 1064 1065 /* Enable BAR2 ALIAS for this pcifunc. */ 1066 reg = BIT_ULL(16) | pcifunc; 1067 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg); 1068 1069 for (i = 0; i < max_ctx_entries; i++) { 1070 cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i)); 1071 1072 if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) && 1073 FIELD_GET(CTX_CAM_CPTR, cam_data)) { 1074 reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data); 1075 rvu_write64(rvu, blkaddr, 1076 CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH), 1077 reg); 1078 } 1079 } 1080 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0); 1081 1082 unlock: 1083 mutex_unlock(&rvu->rsrc_lock); 1084 1085 return 0; 1086 } 1087