1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell OcteonTx2 RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell International Ltd. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/pci.h> 13 14 #include "rvu_struct.h" 15 #include "rvu_reg.h" 16 #include "rvu.h" 17 18 static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block, 19 struct npa_aq_inst_s *inst) 20 { 21 struct admin_queue *aq = block->aq; 22 struct npa_aq_res_s *result; 23 int timeout = 1000; 24 u64 reg, head; 25 26 result = (struct npa_aq_res_s *)aq->res->base; 27 28 /* Get current head pointer where to append this instruction */ 29 reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS); 30 head = (reg >> 4) & AQ_PTR_MASK; 31 32 memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)), 33 (void *)inst, aq->inst->entry_sz); 34 memset(result, 0, sizeof(*result)); 35 /* sync into memory */ 36 wmb(); 37 38 /* Ring the doorbell and wait for result */ 39 rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1); 40 while (result->compcode == NPA_AQ_COMP_NOTDONE) { 41 cpu_relax(); 42 udelay(1); 43 timeout--; 44 if (!timeout) 45 return -EBUSY; 46 } 47 48 if (result->compcode != NPA_AQ_COMP_GOOD) 49 /* TODO: Replace this with some error code */ 50 return -EBUSY; 51 52 return 0; 53 } 54 55 int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req, 56 struct npa_aq_enq_rsp *rsp) 57 { 58 struct rvu_hwinfo *hw = rvu->hw; 59 u16 pcifunc = req->hdr.pcifunc; 60 int blkaddr, npalf, rc = 0; 61 struct npa_aq_inst_s inst; 62 struct rvu_block *block; 63 struct admin_queue *aq; 64 struct rvu_pfvf *pfvf; 65 void *ctx, *mask; 66 bool ena; 67 68 pfvf = rvu_get_pfvf(rvu, pcifunc); 69 if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize) 70 return NPA_AF_ERR_AQ_ENQUEUE; 71 72 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 73 if (!pfvf->npalf || blkaddr < 0) 74 return NPA_AF_ERR_AF_LF_INVALID; 75 76 block = &hw->block[blkaddr]; 77 aq = block->aq; 78 if (!aq) { 79 dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__); 80 return NPA_AF_ERR_AQ_ENQUEUE; 81 } 82 83 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 84 if (npalf < 0) 85 return NPA_AF_ERR_AF_LF_INVALID; 86 87 memset(&inst, 0, sizeof(struct npa_aq_inst_s)); 88 inst.cindex = req->aura_id; 89 inst.lf = npalf; 90 inst.ctype = req->ctype; 91 inst.op = req->op; 92 /* Currently we are not supporting enqueuing multiple instructions, 93 * so always choose first entry in result memory. 94 */ 95 inst.res_addr = (u64)aq->res->iova; 96 97 /* Hardware uses same aq->res->base for updating result of 98 * previous instruction hence wait here till it is done. 99 */ 100 spin_lock(&aq->lock); 101 102 /* Clean result + context memory */ 103 memset(aq->res->base, 0, aq->res->entry_sz); 104 /* Context needs to be written at RES_ADDR + 128 */ 105 ctx = aq->res->base + 128; 106 /* Mask needs to be written at RES_ADDR + 256 */ 107 mask = aq->res->base + 256; 108 109 switch (req->op) { 110 case NPA_AQ_INSTOP_WRITE: 111 /* Copy context and write mask */ 112 if (req->ctype == NPA_AQ_CTYPE_AURA) { 113 memcpy(mask, &req->aura_mask, 114 sizeof(struct npa_aura_s)); 115 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); 116 } else { 117 memcpy(mask, &req->pool_mask, 118 sizeof(struct npa_pool_s)); 119 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); 120 } 121 break; 122 case NPA_AQ_INSTOP_INIT: 123 if (req->ctype == NPA_AQ_CTYPE_AURA) { 124 if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) { 125 rc = NPA_AF_ERR_AQ_FULL; 126 break; 127 } 128 /* Set pool's context address */ 129 req->aura.pool_addr = pfvf->pool_ctx->iova + 130 (req->aura.pool_addr * pfvf->pool_ctx->entry_sz); 131 memcpy(ctx, &req->aura, sizeof(struct npa_aura_s)); 132 } else { /* POOL's context */ 133 memcpy(ctx, &req->pool, sizeof(struct npa_pool_s)); 134 } 135 break; 136 case NPA_AQ_INSTOP_NOP: 137 case NPA_AQ_INSTOP_READ: 138 case NPA_AQ_INSTOP_LOCK: 139 case NPA_AQ_INSTOP_UNLOCK: 140 break; 141 default: 142 rc = NPA_AF_ERR_AQ_FULL; 143 break; 144 } 145 146 if (rc) { 147 spin_unlock(&aq->lock); 148 return rc; 149 } 150 151 /* Submit the instruction to AQ */ 152 rc = npa_aq_enqueue_wait(rvu, block, &inst); 153 if (rc) { 154 spin_unlock(&aq->lock); 155 return rc; 156 } 157 158 /* Set aura bitmap if aura hw context is enabled */ 159 if (req->ctype == NPA_AQ_CTYPE_AURA) { 160 if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena) 161 __set_bit(req->aura_id, pfvf->aura_bmap); 162 if (req->op == NPA_AQ_INSTOP_WRITE) { 163 ena = (req->aura.ena & req->aura_mask.ena) | 164 (test_bit(req->aura_id, pfvf->aura_bmap) & 165 ~req->aura_mask.ena); 166 if (ena) 167 __set_bit(req->aura_id, pfvf->aura_bmap); 168 else 169 __clear_bit(req->aura_id, pfvf->aura_bmap); 170 } 171 } 172 173 /* Set pool bitmap if pool hw context is enabled */ 174 if (req->ctype == NPA_AQ_CTYPE_POOL) { 175 if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena) 176 __set_bit(req->aura_id, pfvf->pool_bmap); 177 if (req->op == NPA_AQ_INSTOP_WRITE) { 178 ena = (req->pool.ena & req->pool_mask.ena) | 179 (test_bit(req->aura_id, pfvf->pool_bmap) & 180 ~req->pool_mask.ena); 181 if (ena) 182 __set_bit(req->aura_id, pfvf->pool_bmap); 183 else 184 __clear_bit(req->aura_id, pfvf->pool_bmap); 185 } 186 } 187 spin_unlock(&aq->lock); 188 189 if (rsp) { 190 /* Copy read context into mailbox */ 191 if (req->op == NPA_AQ_INSTOP_READ) { 192 if (req->ctype == NPA_AQ_CTYPE_AURA) 193 memcpy(&rsp->aura, ctx, 194 sizeof(struct npa_aura_s)); 195 else 196 memcpy(&rsp->pool, ctx, 197 sizeof(struct npa_pool_s)); 198 } 199 } 200 201 return 0; 202 } 203 204 static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req) 205 { 206 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); 207 struct npa_aq_enq_req aq_req; 208 unsigned long *bmap; 209 int id, cnt = 0; 210 int err = 0, rc; 211 212 if (!pfvf->pool_ctx || !pfvf->aura_ctx) 213 return NPA_AF_ERR_AQ_ENQUEUE; 214 215 memset(&aq_req, 0, sizeof(struct npa_aq_enq_req)); 216 aq_req.hdr.pcifunc = req->hdr.pcifunc; 217 218 if (req->ctype == NPA_AQ_CTYPE_POOL) { 219 aq_req.pool.ena = 0; 220 aq_req.pool_mask.ena = 1; 221 cnt = pfvf->pool_ctx->qsize; 222 bmap = pfvf->pool_bmap; 223 } else if (req->ctype == NPA_AQ_CTYPE_AURA) { 224 aq_req.aura.ena = 0; 225 aq_req.aura_mask.ena = 1; 226 aq_req.aura.bp_ena = 0; 227 aq_req.aura_mask.bp_ena = 1; 228 cnt = pfvf->aura_ctx->qsize; 229 bmap = pfvf->aura_bmap; 230 } 231 232 aq_req.ctype = req->ctype; 233 aq_req.op = NPA_AQ_INSTOP_WRITE; 234 235 for (id = 0; id < cnt; id++) { 236 if (!test_bit(id, bmap)) 237 continue; 238 aq_req.aura_id = id; 239 rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL); 240 if (rc) { 241 err = rc; 242 dev_err(rvu->dev, "Failed to disable %s:%d context\n", 243 (req->ctype == NPA_AQ_CTYPE_AURA) ? 244 "Aura" : "Pool", id); 245 } 246 } 247 248 return err; 249 } 250 251 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 252 static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req) 253 { 254 struct npa_aq_enq_req lock_ctx_req; 255 int err; 256 257 if (req->op != NPA_AQ_INSTOP_INIT) 258 return 0; 259 260 memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req)); 261 lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc; 262 lock_ctx_req.ctype = req->ctype; 263 lock_ctx_req.op = NPA_AQ_INSTOP_LOCK; 264 lock_ctx_req.aura_id = req->aura_id; 265 err = rvu_npa_aq_enq_inst(rvu, &lock_ctx_req, NULL); 266 if (err) 267 dev_err(rvu->dev, 268 "PFUNC 0x%x: Failed to lock NPA context %s:%d\n", 269 req->hdr.pcifunc, 270 (req->ctype == NPA_AQ_CTYPE_AURA) ? 271 "Aura" : "Pool", req->aura_id); 272 return err; 273 } 274 275 int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, 276 struct npa_aq_enq_req *req, 277 struct npa_aq_enq_rsp *rsp) 278 { 279 int err; 280 281 err = rvu_npa_aq_enq_inst(rvu, req, rsp); 282 if (!err) 283 err = npa_lf_hwctx_lockdown(rvu, req); 284 return err; 285 } 286 #else 287 288 int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu, 289 struct npa_aq_enq_req *req, 290 struct npa_aq_enq_rsp *rsp) 291 { 292 return rvu_npa_aq_enq_inst(rvu, req, rsp); 293 } 294 #endif 295 296 int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu, 297 struct hwctx_disable_req *req, 298 struct msg_rsp *rsp) 299 { 300 return npa_lf_hwctx_disable(rvu, req); 301 } 302 303 static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf) 304 { 305 kfree(pfvf->aura_bmap); 306 pfvf->aura_bmap = NULL; 307 308 qmem_free(rvu->dev, pfvf->aura_ctx); 309 pfvf->aura_ctx = NULL; 310 311 kfree(pfvf->pool_bmap); 312 pfvf->pool_bmap = NULL; 313 314 qmem_free(rvu->dev, pfvf->pool_ctx); 315 pfvf->pool_ctx = NULL; 316 317 qmem_free(rvu->dev, pfvf->npa_qints_ctx); 318 pfvf->npa_qints_ctx = NULL; 319 } 320 321 int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu, 322 struct npa_lf_alloc_req *req, 323 struct npa_lf_alloc_rsp *rsp) 324 { 325 int npalf, qints, hwctx_size, err, rc = 0; 326 struct rvu_hwinfo *hw = rvu->hw; 327 u16 pcifunc = req->hdr.pcifunc; 328 struct rvu_block *block; 329 struct rvu_pfvf *pfvf; 330 u64 cfg, ctx_cfg; 331 int blkaddr; 332 333 if (req->aura_sz > NPA_AURA_SZ_MAX || 334 req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools) 335 return NPA_AF_ERR_PARAM; 336 337 if (req->way_mask) 338 req->way_mask &= 0xFFFF; 339 340 pfvf = rvu_get_pfvf(rvu, pcifunc); 341 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 342 if (!pfvf->npalf || blkaddr < 0) 343 return NPA_AF_ERR_AF_LF_INVALID; 344 345 block = &hw->block[blkaddr]; 346 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 347 if (npalf < 0) 348 return NPA_AF_ERR_AF_LF_INVALID; 349 350 /* Reset this NPA LF */ 351 err = rvu_lf_reset(rvu, block, npalf); 352 if (err) { 353 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf); 354 return NPA_AF_ERR_LF_RESET; 355 } 356 357 ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1); 358 359 /* Alloc memory for aura HW contexts */ 360 hwctx_size = 1UL << (ctx_cfg & 0xF); 361 err = qmem_alloc(rvu->dev, &pfvf->aura_ctx, 362 NPA_AURA_COUNT(req->aura_sz), hwctx_size); 363 if (err) 364 goto free_mem; 365 366 pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long), 367 GFP_KERNEL); 368 if (!pfvf->aura_bmap) 369 goto free_mem; 370 371 /* Alloc memory for pool HW contexts */ 372 hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF); 373 err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size); 374 if (err) 375 goto free_mem; 376 377 pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long), 378 GFP_KERNEL); 379 if (!pfvf->pool_bmap) 380 goto free_mem; 381 382 /* Get no of queue interrupts supported */ 383 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST); 384 qints = (cfg >> 28) & 0xFFF; 385 386 /* Alloc memory for Qints HW contexts */ 387 hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF); 388 err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size); 389 if (err) 390 goto free_mem; 391 392 cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf)); 393 /* Clear way partition mask and set aura offset to '0' */ 394 cfg &= ~(BIT_ULL(34) - 1); 395 /* Set aura size & enable caching of contexts */ 396 cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask; 397 398 rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg); 399 400 /* Configure aura HW context's base */ 401 rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf), 402 (u64)pfvf->aura_ctx->iova); 403 404 /* Enable caching of qints hw context */ 405 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf), 406 BIT_ULL(36) | req->way_mask << 20); 407 rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf), 408 (u64)pfvf->npa_qints_ctx->iova); 409 410 goto exit; 411 412 free_mem: 413 npa_ctx_free(rvu, pfvf); 414 rc = -ENOMEM; 415 416 exit: 417 /* set stack page info */ 418 cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST); 419 rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF; 420 rsp->stack_pg_bytes = cfg & 0xFF; 421 rsp->qints = (cfg >> 28) & 0xFFF; 422 return rc; 423 } 424 425 int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req, 426 struct msg_rsp *rsp) 427 { 428 struct rvu_hwinfo *hw = rvu->hw; 429 u16 pcifunc = req->hdr.pcifunc; 430 struct rvu_block *block; 431 struct rvu_pfvf *pfvf; 432 int npalf, err; 433 int blkaddr; 434 435 pfvf = rvu_get_pfvf(rvu, pcifunc); 436 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc); 437 if (!pfvf->npalf || blkaddr < 0) 438 return NPA_AF_ERR_AF_LF_INVALID; 439 440 block = &hw->block[blkaddr]; 441 npalf = rvu_get_lf(rvu, block, pcifunc, 0); 442 if (npalf < 0) 443 return NPA_AF_ERR_AF_LF_INVALID; 444 445 /* Reset this NPA LF */ 446 err = rvu_lf_reset(rvu, block, npalf); 447 if (err) { 448 dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf); 449 return NPA_AF_ERR_LF_RESET; 450 } 451 452 npa_ctx_free(rvu, pfvf); 453 454 return 0; 455 } 456 457 static int npa_aq_init(struct rvu *rvu, struct rvu_block *block) 458 { 459 u64 cfg; 460 int err; 461 462 /* Set admin queue endianness */ 463 cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG); 464 #ifdef __BIG_ENDIAN 465 cfg |= BIT_ULL(1); 466 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg); 467 #else 468 cfg &= ~BIT_ULL(1); 469 rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg); 470 #endif 471 472 /* Do not bypass NDC cache */ 473 cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG); 474 cfg &= ~0x03DULL; 475 #ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING 476 /* Disable caching of stack pages */ 477 cfg |= 0x10ULL; 478 #endif 479 rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg); 480 481 /* Result structure can be followed by Aura/Pool context at 482 * RES + 128bytes and a write mask at RES + 256 bytes, depending on 483 * operation type. Alloc sufficient result memory for all operations. 484 */ 485 err = rvu_aq_alloc(rvu, &block->aq, 486 Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s), 487 ALIGN(sizeof(struct npa_aq_res_s), 128) + 256); 488 if (err) 489 return err; 490 491 rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE); 492 rvu_write64(rvu, block->addr, 493 NPA_AF_AQ_BASE, (u64)block->aq->inst->iova); 494 return 0; 495 } 496 497 int rvu_npa_init(struct rvu *rvu) 498 { 499 struct rvu_hwinfo *hw = rvu->hw; 500 int blkaddr; 501 502 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0); 503 if (blkaddr < 0) 504 return 0; 505 506 /* Initialize admin queue */ 507 return npa_aq_init(rvu, &hw->block[blkaddr]); 508 } 509 510 void rvu_npa_freemem(struct rvu *rvu) 511 { 512 struct rvu_hwinfo *hw = rvu->hw; 513 struct rvu_block *block; 514 int blkaddr; 515 516 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0); 517 if (blkaddr < 0) 518 return; 519 520 block = &hw->block[blkaddr]; 521 rvu_aq_free(rvu, block->aq); 522 } 523 524 void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf) 525 { 526 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc); 527 struct hwctx_disable_req ctx_req; 528 529 /* Disable all pools */ 530 ctx_req.hdr.pcifunc = pcifunc; 531 ctx_req.ctype = NPA_AQ_CTYPE_POOL; 532 npa_lf_hwctx_disable(rvu, &ctx_req); 533 534 /* Disable all auras */ 535 ctx_req.ctype = NPA_AQ_CTYPE_AURA; 536 npa_lf_hwctx_disable(rvu, &ctx_req); 537 538 npa_ctx_free(rvu, pfvf); 539 } 540