1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2020 Marvell. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/pci.h> 6 #include "rvu_struct.h" 7 #include "rvu_reg.h" 8 #include "mbox.h" 9 #include "rvu.h" 10 11 /* CPT PF device id */ 12 #define PCI_DEVID_OTX2_CPT_PF 0xA0FD 13 #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2 14 15 /* Length of initial context fetch in 128 byte words */ 16 #define CPT_CTX_ILEN 2 17 18 #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \ 19 ({ \ 20 u64 free_sts = 0, busy_sts = 0; \ 21 typeof(rsp) _rsp = rsp; \ 22 u32 e, i; \ 23 \ 24 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \ 25 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \ 26 if (reg & 0x1) \ 27 busy_sts |= 1ULL << i; \ 28 \ 29 if (reg & 0x2) \ 30 free_sts |= 1ULL << i; \ 31 } \ 32 (_rsp)->busy_sts_##etype = busy_sts; \ 33 (_rsp)->free_sts_##etype = free_sts; \ 34 }) 35 36 static int get_cpt_pf_num(struct rvu *rvu) 37 { 38 int i, domain_nr, cpt_pf_num = -1; 39 struct pci_dev *pdev; 40 41 domain_nr = pci_domain_nr(rvu->pdev->bus); 42 for (i = 0; i < rvu->hw->total_pfs; i++) { 43 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0); 44 if (!pdev) 45 continue; 46 47 if (pdev->device == PCI_DEVID_OTX2_CPT_PF || 48 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) { 49 cpt_pf_num = i; 50 put_device(&pdev->dev); 51 break; 52 } 53 put_device(&pdev->dev); 54 } 55 return cpt_pf_num; 56 } 57 58 static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc) 59 { 60 int cpt_pf_num = get_cpt_pf_num(rvu); 61 62 if (rvu_get_pf(pcifunc) != cpt_pf_num) 63 return false; 64 if (pcifunc & RVU_PFVF_FUNC_MASK) 65 return false; 66 67 return true; 68 } 69 70 static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc) 71 { 72 int cpt_pf_num = get_cpt_pf_num(rvu); 73 74 if (rvu_get_pf(pcifunc) != cpt_pf_num) 75 return false; 76 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) 77 return false; 78 79 return true; 80 } 81 82 static int validate_and_get_cpt_blkaddr(int req_blkaddr) 83 { 84 int blkaddr; 85 86 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0; 87 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1) 88 return -EINVAL; 89 90 return blkaddr; 91 } 92 93 int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu, 94 struct cpt_lf_alloc_req_msg *req, 95 struct msg_rsp *rsp) 96 { 97 u16 pcifunc = req->hdr.pcifunc; 98 struct rvu_block *block; 99 int cptlf, blkaddr; 100 int num_lfs, slot; 101 u64 val; 102 103 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 104 if (blkaddr < 0) 105 return blkaddr; 106 107 if (req->eng_grpmsk == 0x0) 108 return CPT_AF_ERR_GRP_INVALID; 109 110 block = &rvu->hw->block[blkaddr]; 111 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 112 block->addr); 113 if (!num_lfs) 114 return CPT_AF_ERR_LF_INVALID; 115 116 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */ 117 if (req->nix_pf_func) { 118 /* If default, use 'this' CPTLF's PFFUNC */ 119 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC) 120 req->nix_pf_func = pcifunc; 121 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX)) 122 return CPT_AF_ERR_NIX_PF_FUNC_INVALID; 123 } 124 125 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */ 126 if (req->sso_pf_func) { 127 /* If default, use 'this' CPTLF's PFFUNC */ 128 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC) 129 req->sso_pf_func = pcifunc; 130 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO)) 131 return CPT_AF_ERR_SSO_PF_FUNC_INVALID; 132 } 133 134 for (slot = 0; slot < num_lfs; slot++) { 135 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 136 if (cptlf < 0) 137 return CPT_AF_ERR_LF_INVALID; 138 139 /* Set CPT LF group and priority */ 140 val = (u64)req->eng_grpmsk << 48 | 1; 141 if (!is_rvu_otx2(rvu)) 142 val |= (CPT_CTX_ILEN << 17); 143 144 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val); 145 146 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC */ 147 val = (u64)req->nix_pf_func << 48 | 148 (u64)req->sso_pf_func << 32; 149 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val); 150 } 151 152 return 0; 153 } 154 155 static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr) 156 { 157 u16 pcifunc = req->hdr.pcifunc; 158 int num_lfs, cptlf, slot; 159 struct rvu_block *block; 160 161 block = &rvu->hw->block[blkaddr]; 162 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), 163 block->addr); 164 if (!num_lfs) 165 return 0; 166 167 for (slot = 0; slot < num_lfs; slot++) { 168 cptlf = rvu_get_lf(rvu, block, pcifunc, slot); 169 if (cptlf < 0) 170 return CPT_AF_ERR_LF_INVALID; 171 172 /* Reset CPT LF group and priority */ 173 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), 0x0); 174 /* Reset CPT LF NIX_PF_FUNC and SSO_PF_FUNC */ 175 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), 0x0); 176 } 177 178 return 0; 179 } 180 181 int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req, 182 struct msg_rsp *rsp) 183 { 184 int ret; 185 186 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0); 187 if (ret) 188 return ret; 189 190 if (is_block_implemented(rvu->hw, BLKADDR_CPT1)) 191 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1); 192 193 return ret; 194 } 195 196 static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req) 197 { 198 u64 offset = req->reg_offset; 199 int blkaddr, num_lfs, lf; 200 struct rvu_block *block; 201 struct rvu_pfvf *pfvf; 202 203 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 204 if (blkaddr < 0) 205 return blkaddr; 206 207 /* Registers that can be accessed from PF/VF */ 208 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) || 209 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) { 210 if (offset & 7) 211 return false; 212 213 lf = (offset & 0xFFF) >> 3; 214 block = &rvu->hw->block[blkaddr]; 215 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 216 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr); 217 if (lf >= num_lfs) 218 /* Slot is not valid for that PF/VF */ 219 return false; 220 221 /* Translate local LF used by VFs to global CPT LF */ 222 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr], 223 req->hdr.pcifunc, lf); 224 if (lf < 0) 225 return false; 226 227 return true; 228 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) { 229 /* Registers that can be accessed from PF */ 230 switch (offset) { 231 case CPT_AF_CTL: 232 case CPT_AF_PF_FUNC: 233 case CPT_AF_BLK_RST: 234 case CPT_AF_CONSTANTS1: 235 case CPT_AF_CTX_FLUSH_TIMER: 236 return true; 237 } 238 239 switch (offset & 0xFF000) { 240 case CPT_AF_EXEX_STS(0): 241 case CPT_AF_EXEX_CTL(0): 242 case CPT_AF_EXEX_CTL2(0): 243 case CPT_AF_EXEX_UCODE_BASE(0): 244 if (offset & 7) 245 return false; 246 break; 247 default: 248 return false; 249 } 250 return true; 251 } 252 return false; 253 } 254 255 int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu, 256 struct cpt_rd_wr_reg_msg *req, 257 struct cpt_rd_wr_reg_msg *rsp) 258 { 259 int blkaddr; 260 261 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 262 if (blkaddr < 0) 263 return blkaddr; 264 265 /* This message is accepted only if sent from CPT PF/VF */ 266 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 267 !is_cpt_vf(rvu, req->hdr.pcifunc)) 268 return CPT_AF_ERR_ACCESS_DENIED; 269 270 rsp->reg_offset = req->reg_offset; 271 rsp->ret_val = req->ret_val; 272 rsp->is_write = req->is_write; 273 274 if (!is_valid_offset(rvu, req)) 275 return CPT_AF_ERR_ACCESS_DENIED; 276 277 if (req->is_write) 278 rvu_write64(rvu, blkaddr, req->reg_offset, req->val); 279 else 280 rsp->val = rvu_read64(rvu, blkaddr, req->reg_offset); 281 282 return 0; 283 } 284 285 static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 286 { 287 if (is_rvu_otx2(rvu)) 288 return; 289 290 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC); 291 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC); 292 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC); 293 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr, 294 CPT_AF_CTX_AOP_LATENCY_PC); 295 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC); 296 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr, 297 CPT_AF_CTX_IFETCH_LATENCY_PC); 298 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 299 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr, 300 CPT_AF_CTX_FFETCH_LATENCY_PC); 301 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 302 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr, 303 CPT_AF_CTX_FFETCH_LATENCY_PC); 304 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC); 305 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr, 306 CPT_AF_CTX_FFETCH_LATENCY_PC); 307 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR); 308 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID); 309 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER); 310 311 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME); 312 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG); 313 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS); 314 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS); 315 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG); 316 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0)); 317 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1)); 318 } 319 320 static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr) 321 { 322 u16 max_ses, max_ies, max_aes; 323 u32 e_min = 0, e_max = 0; 324 u64 reg; 325 326 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1); 327 max_ses = reg & 0xffff; 328 max_ies = (reg >> 16) & 0xffff; 329 max_aes = (reg >> 32) & 0xffff; 330 331 /* Get AE status */ 332 e_min = max_ses + max_ies; 333 e_max = max_ses + max_ies + max_aes; 334 cpt_get_eng_sts(e_min, e_max, rsp, ae); 335 /* Get SE status */ 336 e_min = 0; 337 e_max = max_ses; 338 cpt_get_eng_sts(e_min, e_max, rsp, se); 339 /* Get IE status */ 340 e_min = max_ses; 341 e_max = max_ses + max_ies; 342 cpt_get_eng_sts(e_min, e_max, rsp, ie); 343 } 344 345 int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req, 346 struct cpt_sts_rsp *rsp) 347 { 348 int blkaddr; 349 350 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 351 if (blkaddr < 0) 352 return blkaddr; 353 354 /* This message is accepted only if sent from CPT PF/VF */ 355 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 356 !is_cpt_vf(rvu, req->hdr.pcifunc)) 357 return CPT_AF_ERR_ACCESS_DENIED; 358 359 get_ctx_pc(rvu, rsp, blkaddr); 360 361 /* Get CPT engines status */ 362 get_eng_sts(rvu, rsp, blkaddr); 363 364 /* Read CPT instruction PC registers */ 365 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC); 366 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC); 367 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC); 368 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC); 369 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC); 370 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr, 371 CPT_AF_ACTIVE_CYCLES_PC); 372 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO); 373 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT); 374 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG); 375 376 return 0; 377 } 378 379 #define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48) 380 #define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32) 381 #define RXC_ACTIVE_THRES GENMASK_ULL(27, 16) 382 #define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0) 383 #define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48) 384 #define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48) 385 386 static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req, 387 int blkaddr) 388 { 389 u64 dfrg_reg; 390 391 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres); 392 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit); 393 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres); 394 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit); 395 396 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step); 397 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg); 398 } 399 400 int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu, 401 struct cpt_rxc_time_cfg_req *req, 402 struct msg_rsp *rsp) 403 { 404 int blkaddr; 405 406 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr); 407 if (blkaddr < 0) 408 return blkaddr; 409 410 /* This message is accepted only if sent from CPT PF/VF */ 411 if (!is_cpt_pf(rvu, req->hdr.pcifunc) && 412 !is_cpt_vf(rvu, req->hdr.pcifunc)) 413 return CPT_AF_ERR_ACCESS_DENIED; 414 415 cpt_rxc_time_cfg(rvu, req, blkaddr); 416 417 return 0; 418 } 419 420 #define INPROG_INFLIGHT(reg) ((reg) & 0x1FF) 421 #define INPROG_GRB_PARTIAL(reg) ((reg) & BIT_ULL(31)) 422 #define INPROG_GRB(reg) (((reg) >> 32) & 0xFF) 423 #define INPROG_GWB(reg) (((reg) >> 40) & 0xFF) 424 425 static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot) 426 { 427 int i = 0, hard_lp_ctr = 100000; 428 u64 inprog, grp_ptr; 429 u16 nq_ptr, dq_ptr; 430 431 /* Disable instructions enqueuing */ 432 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0); 433 434 /* Disable executions in the LF's queue */ 435 inprog = rvu_read64(rvu, blkaddr, 436 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 437 inprog &= ~BIT_ULL(16); 438 rvu_write64(rvu, blkaddr, 439 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog); 440 441 /* Wait for CPT queue to become execution-quiescent */ 442 do { 443 inprog = rvu_read64(rvu, blkaddr, 444 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 445 if (INPROG_GRB_PARTIAL(inprog)) { 446 i = 0; 447 hard_lp_ctr--; 448 } else { 449 i++; 450 } 451 452 grp_ptr = rvu_read64(rvu, blkaddr, 453 CPT_AF_BAR2_ALIASX(slot, 454 CPT_LF_Q_GRP_PTR)); 455 nq_ptr = (grp_ptr >> 32) & 0x7FFF; 456 dq_ptr = grp_ptr & 0x7FFF; 457 458 } while (hard_lp_ctr && (i < 10) && (nq_ptr != dq_ptr)); 459 460 if (hard_lp_ctr == 0) 461 dev_warn(rvu->dev, "CPT FLR hits hard loop counter\n"); 462 463 i = 0; 464 hard_lp_ctr = 100000; 465 do { 466 inprog = rvu_read64(rvu, blkaddr, 467 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 468 469 if ((INPROG_INFLIGHT(inprog) == 0) && 470 (INPROG_GWB(inprog) < 40) && 471 ((INPROG_GRB(inprog) == 0) || 472 (INPROG_GRB((inprog)) == 40))) { 473 i++; 474 } else { 475 i = 0; 476 hard_lp_ctr--; 477 } 478 } while (hard_lp_ctr && (i < 10)); 479 480 if (hard_lp_ctr == 0) 481 dev_warn(rvu->dev, "CPT FLR hits hard loop counter\n"); 482 } 483 484 int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int lf, int slot) 485 { 486 int blkaddr; 487 u64 reg; 488 489 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_CPT, pcifunc); 490 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1) 491 return -EINVAL; 492 493 /* Enable BAR2 ALIAS for this pcifunc. */ 494 reg = BIT_ULL(16) | pcifunc; 495 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg); 496 497 cpt_lf_disable_iqueue(rvu, blkaddr, slot); 498 499 /* Set group drop to help clear out hardware */ 500 reg = rvu_read64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG)); 501 reg |= BIT_ULL(17); 502 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), reg); 503 504 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0); 505 506 return 0; 507 } 508