1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/module.h> 9 #include <linux/pci.h> 10 11 #include "rvu_struct.h" 12 #include "rvu_reg.h" 13 #include "rvu.h" 14 15 static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, 16 struct npa_aq_inst_s *inst) 17 { 18 struct admin_queue *aq = block->aq; 19 struct npa_aq_res_s *result; 20 int timeout = 1000; 21 u64 reg, head; 22 23 result = (struct npa_aq_res_s *)aq->res->base; 24 25 /* Get current head pointer where to append this instruction */ 26 reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS); 27 head = (reg >> 4) & AQ_PTR_MASK; 28 29 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), 30 (void *)inst, aq->inst->entry_sz); 31 memset(result, 0, sizeof(*result)); 32 /* sync into memory */ 33 wmb(); 34 35 /* Ring the doorbell and wait for result */ 36 rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1); 37 while (result->compcode == NPA_AQ_COMP_NOTDONE) { 38 cpu_relax(); 39 udelay(1); 40 timeout--; 41 if (!timeout) 42 return -EBUSY; 43 } 44 45 if (result->compcode != NPA_AQ_COMP_GOOD) 46 /* TODO: Replace this with some error code */ 47 return -EBUSY; 48 49 return 0; 50 } 51 52 int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req, 53 struct npa_aq_enq_rsp *rsp) 54 { 55 struct rvu_hwinfo *hw = rvu->hw; 56 u16 pcifunc = req->hdr.pcifunc; 57 int blkaddr, npalf, rc = 0; 58 struct npa_aq_inst_s inst; 59 struct rvu_block *block; 60 struct admin_queue *aq; 61 struct rvu_pfvf *pfvf; 62 void *ctx, *mask; 63 bool ena; 64 65 pfvf = rvu_get_pfvf(rvu, pcifunc); 66 if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize) 67 return NPA_AF_ERR_AQ_ENQUEUE; 68 69 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 70 if (!pfvf->npalf || blkaddr < 0) 71 return NPA_AF_ERR_AF_LF_INVALID; 72 73 block = &hw->block[blkaddr]; 74 aq = block->aq; 75 if (!aq) { 76 dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__); 77 return NPA_AF_ERR_AQ_ENQUEUE; 78 } 79 80 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 81 if (npalf < 0) 82 return NPA_AF_ERR_AF_LF_INVALID; 83 84 memset(&inst, 0, sizeof(struct npa_aq_inst_s)); 85 inst.cindex = req->aura_id; 86 inst.lf = npalf; 87 inst.ctype = req->ctype; 88 inst.op = req->op; 89 /* Currently we are not supporting enqueuing multiple instructions, 90 * so always choose first entry in result memory. 91 */ 92 inst.res_addr = (u64)aq->res->iova; 93 94 /* Hardware uses same aq->res->base for updating result of 95 * previous instruction hence wait here till it is done. 96 */ 97 spin_lock(&aq->lock); 98 99 /* Clean result + context memory */ 100 memset(aq->res->base, 0, aq->res->entry_sz); 101 /* Context needs to be written at RES_ADDR + 128 */ 102 ctx = aq->res->base + 128; 103 /* Mask needs to be written at RES_ADDR + 256 */ 104 mask = aq->res->base + 256; 105 106 switch (req->op) { 107 case NPA_AQ_INSTOP_WRITE: 108 /* Copy context and write mask */ 109 if (req->ctype == NPA_AQ_CTYPE_AURA) { 110 memcpy(mask, &req->aura_mask, 111 sizeof(struct npa_aura_s)); 112 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); 113 } else { 114 memcpy(mask, &req->pool_mask, 115 sizeof(struct npa_pool_s)); 116 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); 117 } 118 break; 119 case NPA_AQ_INSTOP_INIT: 120 if (req->ctype == NPA_AQ_CTYPE_AURA) { 121 if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) { 122 rc = NPA_AF_ERR_AQ_FULL; 123 break; 124 } 125 /* Set pool's context address */ 126 req->aura.pool_addr = pfvf->pool_ctx->iova + 127 (req->aura.pool_addr * pfvf->pool_ctx->entry_sz); 128 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); 129 } else { /* POOL's context */ 130 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); 131 } 132 break; 133 case NPA_AQ_INSTOP_NOP: 134 case NPA_AQ_INSTOP_READ: 135 case NPA_AQ_INSTOP_LOCK: 136 case NPA_AQ_INSTOP_UNLOCK: 137 break; 138 default: 139 rc = NPA_AF_ERR_AQ_FULL; 140 break; 141 } 142 143 if (rc) { 144 spin_unlock(&aq->lock); 145 return rc; 146 } 147 148 /* Submit the instruction to AQ */ 149 rc = npa_aq_enqueue_wait(rvu, block, &inst); 150 if (rc) { 151 spin_unlock(&aq->lock); 152 return rc; 153 } 154 155 /* Set aura bitmap if aura hw context is enabled */ 156 if (req->ctype == NPA_AQ_CTYPE_AURA) { 157 if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena) 158 __set_bit(req->aura_id, pfvf->aura_bmap); 159 if (req->op == NPA_AQ_INSTOP_WRITE) { 160 ena = (req->aura.ena & req->aura_mask.ena) | 161 (test_bit(req->aura_id, pfvf->aura_bmap) & 162 ~req->aura_mask.ena); 163 if (ena) 164 __set_bit(req->aura_id, pfvf->aura_bmap); 165 else 166 __clear_bit(req->aura_id, pfvf->aura_bmap); 167 } 168 } 169 170 /* Set pool bitmap if pool hw context is enabled */ 171 if (req->ctype == NPA_AQ_CTYPE_POOL) { 172 if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena) 173 __set_bit(req->aura_id, pfvf->pool_bmap); 174 if (req->op == NPA_AQ_INSTOP_WRITE) { 175 ena = (req->pool.ena & req->pool_mask.ena) | 176 (test_bit(req->aura_id, pfvf->pool_bmap) & 177 ~req->pool_mask.ena); 178 if (ena) 179 __set_bit(req->aura_id, pfvf->pool_bmap); 180 else 181 __clear_bit(req->aura_id, pfvf->pool_bmap); 182 } 183 } 184 spin_unlock(&aq->lock); 185 186 if (rsp) { 187 /* Copy read context into mailbox */ 188 if (req->op == NPA_AQ_INSTOP_READ) { 189 if (req->ctype == NPA_AQ_CTYPE_AURA) 190 memcpy(&rsp->aura, ctx, 191 sizeof(struct npa_aura_s)); 192 else 193 memcpy(&rsp->pool, ctx, 194 sizeof(struct npa_pool_s)); 195 } 196 } 197 198 return 0; 199 } 200 201 static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) 202 { 203 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 204 struct npa_aq_enq_req aq_req; 205 unsigned long *bmap; 206 int id, cnt = 0; 207 int err = 0, rc; 208 209 if (!pfvf->pool_ctx || !pfvf->aura_ctx) 210 return NPA_AF_ERR_AQ_ENQUEUE; 211 212 memset(&aq_req, 0, sizeof(struct npa_aq_enq_req)); 213 aq_req.hdr.pcifunc = req->hdr.pcifunc; 214 215 if (req->ctype == NPA_AQ_CTYPE_POOL) { 216 aq_req.pool.ena = 0; 217 aq_req.pool_mask.ena = 1; 218 cnt = pfvf->pool_ctx->qsize; 219 bmap = pfvf->pool_bmap; 220 } else if (req->ctype == NPA_AQ_CTYPE_AURA) { 221 aq_req.aura.ena = 0; 222 aq_req.aura_mask.ena = 1; 223 aq_req.aura.bp_ena = 0; 224 aq_req.aura_mask.bp_ena = 1; 225 cnt = pfvf->aura_ctx->qsize; 226 bmap = pfvf->aura_bmap; 227 } 228 229 aq_req.ctype = req->ctype; 230 aq_req.op = NPA_AQ_INSTOP_WRITE; 231 232 for (id = 0; id < cnt; id++) { 233 if (!test_bit(id, bmap)) 234 continue; 235 aq_req.aura_id = id; 236 rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL); 237 if (rc) { 238 err = rc; 239 dev_err(rvu->dev, "Failed to disable %s:%d context\n", 240 (req->ctype == NPA_AQ_CTYPE_AURA) ? 241 "Aura" : "Pool", id); 242 } 243 } 244 245 return err; 246 } 247 248 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 249 static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req) 250 { 251 struct npa_aq_enq_req lock_ctx_req; 252 int err; 253 254 if (req->op != NPA_AQ_INSTOP_INIT) 255 return 0; 256 257 memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req)); 258 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; 259 lock_ctx_req.ctype = req->ctype; 260 lock_ctx_req.op = NPA_AQ_INSTOP_LOCK; 261 lock_ctx_req.aura_id = req->aura_id; 262 err = rvu_npa_aq_enq_inst(rvu, &lock_ctx_req, NULL); 263 if (err) 264 dev_err(rvu->dev, 265 "PFUNC 0x%x: Failed to lock NPA context %s:%d\n", 266 req->hdr.pcifunc, 267 (req->ctype == NPA_AQ_CTYPE_AURA) ? 268 "Aura" : "Pool", req->aura_id); 269 return err; 270 } 271 272 int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, 273 struct npa_aq_enq_req *req, 274 struct npa_aq_enq_rsp *rsp) 275 { 276 int err; 277 278 err = rvu_npa_aq_enq_inst(rvu, req, rsp); 279 if (!err) 280 err = npa_lf_hwctx_lockdown(rvu, req); 281 return err; 282 } 283 #else 284 285 int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, 286 struct npa_aq_enq_req *req, 287 struct npa_aq_enq_rsp *rsp) 288 { 289 return rvu_npa_aq_enq_inst(rvu, req, rsp); 290 } 291 #endif 292 293 int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu, 294 struct hwctx_disable_req *req, 295 struct msg_rsp *rsp) 296 { 297 return npa_lf_hwctx_disable(rvu, req); 298 } 299 300 static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) 301 { 302 kfree(pfvf->aura_bmap); 303 pfvf->aura_bmap = NULL; 304 305 qmem_free(rvu->dev, pfvf->aura_ctx); 306 pfvf->aura_ctx = NULL; 307 308 kfree(pfvf->pool_bmap); 309 pfvf->pool_bmap = NULL; 310 311 qmem_free(rvu->dev, pfvf->pool_ctx); 312 pfvf->pool_ctx = NULL; 313 314 qmem_free(rvu->dev, pfvf->npa_qints_ctx); 315 pfvf->npa_qints_ctx = NULL; 316 } 317 318 int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu, 319 struct npa_lf_alloc_req *req, 320 struct npa_lf_alloc_rsp *rsp) 321 { 322 int npalf, qints, hwctx_size, err, rc = 0; 323 struct rvu_hwinfo *hw = rvu->hw; 324 u16 pcifunc = req->hdr.pcifunc; 325 struct rvu_block *block; 326 struct rvu_pfvf *pfvf; 327 u64 cfg, ctx_cfg; 328 int blkaddr; 329 330 if (req->aura_sz > NPA_AURA_SZ_MAX || 331 req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools) 332 return NPA_AF_ERR_PARAM; 333 334 if (req->way_mask) 335 req->way_mask &= 0xFFFF; 336 337 pfvf = rvu_get_pfvf(rvu, pcifunc); 338 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 339 if (!pfvf->npalf || blkaddr < 0) 340 return NPA_AF_ERR_AF_LF_INVALID; 341 342 block = &hw->block[blkaddr]; 343 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 344 if (npalf < 0) 345 return NPA_AF_ERR_AF_LF_INVALID; 346 347 /* Reset this NPA LF */ 348 err = rvu_lf_reset(rvu, block, npalf); 349 if (err) { 350 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf); 351 return NPA_AF_ERR_LF_RESET; 352 } 353 354 ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1); 355 356 /* Alloc memory for aura HW contexts */ 357 hwctx_size = 1UL << (ctx_cfg & 0xF); 358 err = qmem_alloc(rvu->dev, &pfvf->aura_ctx, 359 NPA_AURA_COUNT(req->aura_sz), hwctx_size); 360 if (err) 361 goto free_mem; 362 363 pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long), 364 GFP_KERNEL); 365 if (!pfvf->aura_bmap) 366 goto free_mem; 367 368 /* Alloc memory for pool HW contexts */ 369 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); 370 err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size); 371 if (err) 372 goto free_mem; 373 374 pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long), 375 GFP_KERNEL); 376 if (!pfvf->pool_bmap) 377 goto free_mem; 378 379 /* Get no of queue interrupts supported */ 380 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST); 381 qints = (cfg >> 28) & 0xFFF; 382 383 /* Alloc memory for Qints HW contexts */ 384 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); 385 err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size); 386 if (err) 387 goto free_mem; 388 389 cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf)); 390 /* Clear way partition mask and set aura offset to '0' */ 391 cfg &= ~(BIT_ULL(34) - 1); 392 /* Set aura size & enable caching of contexts */ 393 cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask; 394 395 rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg); 396 397 /* Configure aura HW context's base */ 398 rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf), 399 (u64)pfvf->aura_ctx->iova); 400 401 /* Enable caching of qints hw context */ 402 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf), 403 BIT_ULL(36) | req->way_mask << 20); 404 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf), 405 (u64)pfvf->npa_qints_ctx->iova); 406 407 goto exit; 408 409 free_mem: 410 npa_ctx_free(rvu, pfvf); 411 rc = -ENOMEM; 412 413 exit: 414 /* set stack page info */ 415 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST); 416 rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF; 417 rsp->stack_pg_bytes = cfg & 0xFF; 418 rsp->qints = (cfg >> 28) & 0xFFF; 419 if (!is_rvu_otx2(rvu)) { 420 cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL); 421 rsp->cache_lines = (cfg >> 1) & 0x3F; 422 } 423 return rc; 424 } 425 426 int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req, 427 struct msg_rsp *rsp) 428 { 429 struct rvu_hwinfo *hw = rvu->hw; 430 u16 pcifunc = req->hdr.pcifunc; 431 struct rvu_block *block; 432 struct rvu_pfvf *pfvf; 433 int npalf, err; 434 int blkaddr; 435 436 pfvf = rvu_get_pfvf(rvu, pcifunc); 437 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 438 if (!pfvf->npalf || blkaddr < 0) 439 return NPA_AF_ERR_AF_LF_INVALID; 440 441 block = &hw->block[blkaddr]; 442 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 443 if (npalf < 0) 444 return NPA_AF_ERR_AF_LF_INVALID; 445 446 /* Reset this NPA LF */ 447 err = rvu_lf_reset(rvu, block, npalf); 448 if (err) { 449 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf); 450 return NPA_AF_ERR_LF_RESET; 451 } 452 453 npa_ctx_free(rvu, pfvf); 454 455 return 0; 456 } 457 458 static int npa_aq_init(struct rvu *rvu, struct rvu_block *block) 459 { 460 u64 cfg; 461 int err; 462 463 /* Set admin queue endianness */ 464 cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG); 465 #ifdef __BIG_ENDIAN 466 cfg |= BIT_ULL(1); 467 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg); 468 #else 469 cfg &= ~BIT_ULL(1); 470 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg); 471 #endif 472 473 /* Do not bypass NDC cache */ 474 cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG); 475 cfg &= ~0x03DULL; 476 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 477 /* Disable caching of stack pages */ 478 cfg |= 0x10ULL; 479 #endif 480 rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg); 481 482 /* For CN10K NPA BATCH DMA set 35 cache lines */ 483 if (!is_rvu_otx2(rvu)) { 484 cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL); 485 cfg &= ~0x7EULL; 486 cfg |= BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1); 487 rvu_write64(rvu, block->addr, NPA_AF_BATCH_CTL, cfg); 488 } 489 /* Result structure can be followed by Aura/Pool context at 490 * RES + 128bytes and a write mask at RES + 256 bytes, depending on 491 * operation type. Alloc sufficient result memory for all operations. 492 */ 493 err = rvu_aq_alloc(rvu, &block->aq, 494 Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s), 495 ALIGN(sizeof(struct npa_aq_res_s), 128) + 256); 496 if (err) 497 return err; 498 499 rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE); 500 rvu_write64(rvu, block->addr, 501 NPA_AF_AQ_BASE, (u64)block->aq->inst->iova); 502 return 0; 503 } 504 505 int rvu_npa_init(struct rvu *rvu) 506 { 507 struct rvu_hwinfo *hw = rvu->hw; 508 int blkaddr; 509 510 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0); 511 if (blkaddr < 0) 512 return 0; 513 514 /* Initialize admin queue */ 515 return npa_aq_init(rvu, &hw->block[blkaddr]); 516 } 517 518 void rvu_npa_freemem(struct rvu *rvu) 519 { 520 struct rvu_hwinfo *hw = rvu->hw; 521 struct rvu_block *block; 522 int blkaddr; 523 524 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0); 525 if (blkaddr < 0) 526 return; 527 528 block = &hw->block[blkaddr]; 529 rvu_aq_free(rvu, block->aq); 530 } 531 532 void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf) 533 { 534 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 535 struct hwctx_disable_req ctx_req; 536 537 /* Disable all pools */ 538 ctx_req.hdr.pcifunc = pcifunc; 539 ctx_req.ctype = NPA_AQ_CTYPE_POOL; 540 npa_lf_hwctx_disable(rvu, &ctx_req); 541 542 /* Disable all auras */ 543 ctx_req.ctype = NPA_AQ_CTYPE_AURA; 544 npa_lf_hwctx_disable(rvu, &ctx_req); 545 546 npa_ctx_free(rvu, pfvf); 547 } 548