1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Marvell RVU Admin Function driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/pci.h>
10 #include "rvu_struct.h"
11 #include "rvu_reg.h"
12 #include "mbox.h"
13 #include "rvu.h"
14
15 /* CPT PF device id */
16 #define PCI_DEVID_OTX2_CPT_PF 0xA0FD
17 #define PCI_DEVID_OTX2_CPT10K_PF 0xA0F2
18
19 /* Length of initial context fetch in 128 byte words */
20 #define CPT_CTX_ILEN 1ULL
21
22 #define cpt_get_eng_sts(e_min, e_max, rsp, etype) \
23 ({ \
24 u64 free_sts = 0, busy_sts = 0; \
25 typeof(rsp) _rsp = rsp; \
26 u32 e, i; \
27 \
28 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \
29 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \
30 if (reg & 0x1) \
31 busy_sts |= 1ULL << i; \
32 \
33 if (reg & 0x2) \
34 free_sts |= 1ULL << i; \
35 } \
36 (_rsp)->busy_sts_##etype = busy_sts; \
37 (_rsp)->free_sts_##etype = free_sts; \
38 })
39
cpt_af_flt_intr_handler(int vec,void * ptr)40 static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr)
41 {
42 struct rvu_block *block = ptr;
43 struct rvu *rvu = block->rvu;
44 int blkaddr = block->addr;
45 u64 reg, val;
46 int i, eng;
47 u8 grp;
48
49 reg = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(vec));
50 dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx", vec, reg);
51
52 i = -1;
53 while ((i = find_next_bit((unsigned long *)®, 64, i + 1)) < 64) {
54 switch (vec) {
55 case 0:
56 eng = i;
57 break;
58 case 1:
59 eng = i + 64;
60 break;
61 case 2:
62 eng = i + 128;
63 break;
64 }
65 grp = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF;
66 /* Disable and enable the engine which triggers fault */
67 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), 0x0);
68 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng));
69 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val & ~1ULL);
70
71 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), grp);
72 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val | 1ULL);
73
74 spin_lock(&rvu->cpt_intr_lock);
75 block->cpt_flt_eng_map[vec] |= BIT_ULL(i);
76 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(eng));
77 val = val & 0x3;
78 if (val == 0x1 || val == 0x2)
79 block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i);
80 spin_unlock(&rvu->cpt_intr_lock);
81 }
82 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(vec), reg);
83
84 return IRQ_HANDLED;
85 }
86
rvu_cpt_af_flt0_intr_handler(int irq,void * ptr)87 static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr)
88 {
89 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT0, ptr);
90 }
91
rvu_cpt_af_flt1_intr_handler(int irq,void * ptr)92 static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr)
93 {
94 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT1, ptr);
95 }
96
rvu_cpt_af_flt2_intr_handler(int irq,void * ptr)97 static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr)
98 {
99 return cpt_af_flt_intr_handler(CPT_10K_AF_INT_VEC_FLT2, ptr);
100 }
101
rvu_cpt_af_rvu_intr_handler(int irq,void * ptr)102 static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr)
103 {
104 struct rvu_block *block = ptr;
105 struct rvu *rvu = block->rvu;
106 int blkaddr = block->addr;
107 u64 reg;
108
109 reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT);
110 dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg);
111
112 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg);
113 return IRQ_HANDLED;
114 }
115
rvu_cpt_af_ras_intr_handler(int irq,void * ptr)116 static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr)
117 {
118 struct rvu_block *block = ptr;
119 struct rvu *rvu = block->rvu;
120 int blkaddr = block->addr;
121 u64 reg;
122
123 reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT);
124 dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg);
125
126 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg);
127 return IRQ_HANDLED;
128 }
129
rvu_cpt_do_register_interrupt(struct rvu_block * block,int irq_offs,irq_handler_t handler,const char * name)130 static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs,
131 irq_handler_t handler,
132 const char *name)
133 {
134 struct rvu *rvu = block->rvu;
135 int ret;
136
137 ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0,
138 name, block);
139 if (ret) {
140 dev_err(rvu->dev, "RVUAF: %s irq registration failed", name);
141 return ret;
142 }
143
144 WARN_ON(rvu->irq_allocated[irq_offs]);
145 rvu->irq_allocated[irq_offs] = true;
146 return 0;
147 }
148
cpt_10k_unregister_interrupts(struct rvu_block * block,int off)149 static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off)
150 {
151 struct rvu *rvu = block->rvu;
152 int blkaddr = block->addr;
153 int i;
154
155 /* Disable all CPT AF interrupts */
156 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(0), ~0ULL);
157 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(1), ~0ULL);
158 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(2), 0xFFFF);
159
160 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
161 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
162
163 for (i = 0; i < CPT_10K_AF_INT_VEC_CNT; i++)
164 if (rvu->irq_allocated[off + i]) {
165 free_irq(pci_irq_vector(rvu->pdev, off + i), block);
166 rvu->irq_allocated[off + i] = false;
167 }
168 }
169
cpt_unregister_interrupts(struct rvu * rvu,int blkaddr)170 static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr)
171 {
172 struct rvu_hwinfo *hw = rvu->hw;
173 struct rvu_block *block;
174 int i, offs;
175
176 if (!is_block_implemented(rvu->hw, blkaddr))
177 return;
178 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
179 if (!offs) {
180 dev_warn(rvu->dev,
181 "Failed to get CPT_AF_INT vector offsets\n");
182 return;
183 }
184 block = &hw->block[blkaddr];
185 if (!is_rvu_otx2(rvu))
186 return cpt_10k_unregister_interrupts(block, offs);
187
188 /* Disable all CPT AF interrupts */
189 for (i = 0; i < CPT_AF_INT_VEC_RVU; i++)
190 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), ~0ULL);
191 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
192 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
193
194 for (i = 0; i < CPT_AF_INT_VEC_CNT; i++)
195 if (rvu->irq_allocated[offs + i]) {
196 free_irq(pci_irq_vector(rvu->pdev, offs + i), block);
197 rvu->irq_allocated[offs + i] = false;
198 }
199 }
200
rvu_cpt_unregister_interrupts(struct rvu * rvu)201 void rvu_cpt_unregister_interrupts(struct rvu *rvu)
202 {
203 cpt_unregister_interrupts(rvu, BLKADDR_CPT0);
204 cpt_unregister_interrupts(rvu, BLKADDR_CPT1);
205 }
206
cpt_10k_register_interrupts(struct rvu_block * block,int off)207 static int cpt_10k_register_interrupts(struct rvu_block *block, int off)
208 {
209 struct rvu *rvu = block->rvu;
210 int blkaddr = block->addr;
211 irq_handler_t flt_fn;
212 int i, ret;
213
214 for (i = CPT_10K_AF_INT_VEC_FLT0; i < CPT_10K_AF_INT_VEC_RVU; i++) {
215 sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i);
216
217 switch (i) {
218 case CPT_10K_AF_INT_VEC_FLT0:
219 flt_fn = rvu_cpt_af_flt0_intr_handler;
220 break;
221 case CPT_10K_AF_INT_VEC_FLT1:
222 flt_fn = rvu_cpt_af_flt1_intr_handler;
223 break;
224 case CPT_10K_AF_INT_VEC_FLT2:
225 flt_fn = rvu_cpt_af_flt2_intr_handler;
226 break;
227 }
228 ret = rvu_cpt_do_register_interrupt(block, off + i,
229 flt_fn, &rvu->irq_name[(off + i) * NAME_SIZE]);
230 if (ret)
231 goto err;
232 if (i == CPT_10K_AF_INT_VEC_FLT2)
233 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 0xFFFF);
234 else
235 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);
236 }
237
238 ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RVU,
239 rvu_cpt_af_rvu_intr_handler,
240 "CPTAF RVU");
241 if (ret)
242 goto err;
243 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
244
245 ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RAS,
246 rvu_cpt_af_ras_intr_handler,
247 "CPTAF RAS");
248 if (ret)
249 goto err;
250 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
251
252 return 0;
253 err:
254 rvu_cpt_unregister_interrupts(rvu);
255 return ret;
256 }
257
cpt_register_interrupts(struct rvu * rvu,int blkaddr)258 static int cpt_register_interrupts(struct rvu *rvu, int blkaddr)
259 {
260 struct rvu_hwinfo *hw = rvu->hw;
261 struct rvu_block *block;
262 irq_handler_t flt_fn;
263 int i, offs, ret = 0;
264
265 if (!is_block_implemented(rvu->hw, blkaddr))
266 return 0;
267
268 block = &hw->block[blkaddr];
269 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
270 if (!offs) {
271 dev_warn(rvu->dev,
272 "Failed to get CPT_AF_INT vector offsets\n");
273 return 0;
274 }
275
276 if (!is_rvu_otx2(rvu))
277 return cpt_10k_register_interrupts(block, offs);
278
279 for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) {
280 sprintf(&rvu->irq_name[(offs + i) * NAME_SIZE], "CPTAF FLT%d", i);
281 switch (i) {
282 case CPT_AF_INT_VEC_FLT0:
283 flt_fn = rvu_cpt_af_flt0_intr_handler;
284 break;
285 case CPT_AF_INT_VEC_FLT1:
286 flt_fn = rvu_cpt_af_flt1_intr_handler;
287 break;
288 }
289 ret = rvu_cpt_do_register_interrupt(block, offs + i,
290 flt_fn, &rvu->irq_name[(offs + i) * NAME_SIZE]);
291 if (ret)
292 goto err;
293 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);
294 }
295
296 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU,
297 rvu_cpt_af_rvu_intr_handler,
298 "CPTAF RVU");
299 if (ret)
300 goto err;
301 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
302
303 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS,
304 rvu_cpt_af_ras_intr_handler,
305 "CPTAF RAS");
306 if (ret)
307 goto err;
308 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
309
310 return 0;
311 err:
312 rvu_cpt_unregister_interrupts(rvu);
313 return ret;
314 }
315
rvu_cpt_register_interrupts(struct rvu * rvu)316 int rvu_cpt_register_interrupts(struct rvu *rvu)
317 {
318 int ret;
319
320 ret = cpt_register_interrupts(rvu, BLKADDR_CPT0);
321 if (ret)
322 return ret;
323
324 return cpt_register_interrupts(rvu, BLKADDR_CPT1);
325 }
326
get_cpt_pf_num(struct rvu * rvu)327 static int get_cpt_pf_num(struct rvu *rvu)
328 {
329 int i, domain_nr, cpt_pf_num = -1;
330 struct pci_dev *pdev;
331
332 domain_nr = pci_domain_nr(rvu->pdev->bus);
333 for (i = 0; i < rvu->hw->total_pfs; i++) {
334 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0);
335 if (!pdev)
336 continue;
337
338 if (pdev->device == PCI_DEVID_OTX2_CPT_PF ||
339 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) {
340 cpt_pf_num = i;
341 put_device(&pdev->dev);
342 break;
343 }
344 put_device(&pdev->dev);
345 }
346 return cpt_pf_num;
347 }
348
is_cpt_pf(struct rvu * rvu,u16 pcifunc)349 static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc)
350 {
351 int cpt_pf_num = rvu->cpt_pf_num;
352
353 if (rvu_get_pf(pcifunc) != cpt_pf_num)
354 return false;
355 if (pcifunc & RVU_PFVF_FUNC_MASK)
356 return false;
357
358 return true;
359 }
360
is_cpt_vf(struct rvu * rvu,u16 pcifunc)361 static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc)
362 {
363 int cpt_pf_num = rvu->cpt_pf_num;
364
365 if (rvu_get_pf(pcifunc) != cpt_pf_num)
366 return false;
367 if (!(pcifunc & RVU_PFVF_FUNC_MASK))
368 return false;
369
370 return true;
371 }
372
validate_and_get_cpt_blkaddr(int req_blkaddr)373 static int validate_and_get_cpt_blkaddr(int req_blkaddr)
374 {
375 int blkaddr;
376
377 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0;
378 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)
379 return -EINVAL;
380
381 return blkaddr;
382 }
383
rvu_mbox_handler_cpt_lf_alloc(struct rvu * rvu,struct cpt_lf_alloc_req_msg * req,struct msg_rsp * rsp)384 int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu,
385 struct cpt_lf_alloc_req_msg *req,
386 struct msg_rsp *rsp)
387 {
388 u16 pcifunc = req->hdr.pcifunc;
389 struct rvu_block *block;
390 int cptlf, blkaddr;
391 int num_lfs, slot;
392 u64 val;
393
394 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
395 if (blkaddr < 0)
396 return blkaddr;
397
398 if (req->eng_grpmsk == 0x0)
399 return CPT_AF_ERR_GRP_INVALID;
400
401 block = &rvu->hw->block[blkaddr];
402 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
403 block->addr);
404 if (!num_lfs)
405 return CPT_AF_ERR_LF_INVALID;
406
407 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
408 if (req->nix_pf_func) {
409 /* If default, use 'this' CPTLF's PFFUNC */
410 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC)
411 req->nix_pf_func = pcifunc;
412 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX))
413 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
414 }
415
416 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
417 if (req->sso_pf_func) {
418 /* If default, use 'this' CPTLF's PFFUNC */
419 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC)
420 req->sso_pf_func = pcifunc;
421 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO))
422 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
423 }
424
425 for (slot = 0; slot < num_lfs; slot++) {
426 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
427 if (cptlf < 0)
428 return CPT_AF_ERR_LF_INVALID;
429
430 /* Set CPT LF group and priority */
431 val = (u64)req->eng_grpmsk << 48 | 1;
432 if (!is_rvu_otx2(rvu)) {
433 if (req->ctx_ilen_valid)
434 val |= (req->ctx_ilen << 17);
435 else
436 val |= (CPT_CTX_ILEN << 17);
437 }
438
439 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
440
441 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set
442 * on reset.
443 */
444 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
445 val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32));
446 val |= ((u64)req->nix_pf_func << 48 |
447 (u64)req->sso_pf_func << 32);
448 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
449 }
450
451 return 0;
452 }
453
cpt_lf_free(struct rvu * rvu,struct msg_req * req,int blkaddr)454 static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr)
455 {
456 u16 pcifunc = req->hdr.pcifunc;
457 int num_lfs, cptlf, slot, err;
458 struct rvu_block *block;
459
460 block = &rvu->hw->block[blkaddr];
461 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
462 block->addr);
463 if (!num_lfs)
464 return 0;
465
466 for (slot = 0; slot < num_lfs; slot++) {
467 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
468 if (cptlf < 0)
469 return CPT_AF_ERR_LF_INVALID;
470
471 /* Perform teardown */
472 rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot);
473
474 /* Reset LF */
475 err = rvu_lf_reset(rvu, block, cptlf);
476 if (err) {
477 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
478 block->addr, cptlf);
479 }
480 }
481
482 return 0;
483 }
484
rvu_mbox_handler_cpt_lf_free(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)485 int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req,
486 struct msg_rsp *rsp)
487 {
488 int ret;
489
490 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0);
491 if (ret)
492 return ret;
493
494 if (is_block_implemented(rvu->hw, BLKADDR_CPT1))
495 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1);
496
497 return ret;
498 }
499
cpt_inline_ipsec_cfg_inbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)500 static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf,
501 struct cpt_inline_ipsec_cfg_msg *req)
502 {
503 u16 sso_pf_func = req->sso_pf_func;
504 u8 nix_sel;
505 u64 val;
506
507 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
508 if (req->enable && (val & BIT_ULL(16))) {
509 /* IPSec inline outbound path is already enabled for a given
510 * CPT LF, HRM states that inline inbound & outbound paths
511 * must not be enabled at the same time for a given CPT LF
512 */
513 return CPT_AF_ERR_INLINE_IPSEC_INB_ENA;
514 }
515 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
516 if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO))
517 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
518
519 nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0;
520 /* Enable CPT LF for IPsec inline inbound operations */
521 if (req->enable)
522 val |= BIT_ULL(9);
523 else
524 val &= ~BIT_ULL(9);
525
526 val |= (u64)nix_sel << 8;
527 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
528
529 if (sso_pf_func) {
530 /* Set SSO_PF_FUNC */
531 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
532 val |= (u64)sso_pf_func << 32;
533 val |= (u64)req->nix_pf_func << 48;
534 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
535 }
536 if (req->sso_pf_func_ovrd)
537 /* Set SSO_PF_FUNC_OVRD for inline IPSec */
538 rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1);
539
540 /* Configure the X2P Link register with the cpt base channel number and
541 * range of channels it should propagate to X2P
542 */
543 if (!is_rvu_otx2(rvu)) {
544 val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16);
545 val |= (u64)rvu->hw->cpt_chan_base;
546
547 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val);
548 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val);
549 }
550
551 return 0;
552 }
553
cpt_inline_ipsec_cfg_outbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)554 static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf,
555 struct cpt_inline_ipsec_cfg_msg *req)
556 {
557 u16 nix_pf_func = req->nix_pf_func;
558 int nix_blkaddr;
559 u8 nix_sel;
560 u64 val;
561
562 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
563 if (req->enable && (val & BIT_ULL(9))) {
564 /* IPSec inline inbound path is already enabled for a given
565 * CPT LF, HRM states that inline inbound & outbound paths
566 * must not be enabled at the same time for a given CPT LF
567 */
568 return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA;
569 }
570
571 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
572 if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX))
573 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
574
575 /* Enable CPT LF for IPsec inline outbound operations */
576 if (req->enable)
577 val |= BIT_ULL(16);
578 else
579 val &= ~BIT_ULL(16);
580 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
581
582 if (nix_pf_func) {
583 /* Set NIX_PF_FUNC */
584 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
585 val |= (u64)nix_pf_func << 48;
586 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
587
588 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func);
589 nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1;
590
591 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
592 val |= (u64)nix_sel << 8;
593 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
594 }
595
596 return 0;
597 }
598
rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu * rvu,struct cpt_inline_ipsec_cfg_msg * req,struct msg_rsp * rsp)599 int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu,
600 struct cpt_inline_ipsec_cfg_msg *req,
601 struct msg_rsp *rsp)
602 {
603 u16 pcifunc = req->hdr.pcifunc;
604 struct rvu_block *block;
605 int cptlf, blkaddr, ret;
606 u16 actual_slot;
607
608 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
609 req->slot, &actual_slot);
610 if (blkaddr < 0)
611 return CPT_AF_ERR_LF_INVALID;
612
613 block = &rvu->hw->block[blkaddr];
614
615 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
616 if (cptlf < 0)
617 return CPT_AF_ERR_LF_INVALID;
618
619 switch (req->dir) {
620 case CPT_INLINE_INBOUND:
621 ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req);
622 break;
623
624 case CPT_INLINE_OUTBOUND:
625 ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req);
626 break;
627
628 default:
629 return CPT_AF_ERR_PARAM;
630 }
631
632 return ret;
633 }
634
validate_and_update_reg_offset(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req,u64 * reg_offset)635 static bool validate_and_update_reg_offset(struct rvu *rvu,
636 struct cpt_rd_wr_reg_msg *req,
637 u64 *reg_offset)
638 {
639 u64 offset = req->reg_offset;
640 int blkaddr, num_lfs, lf;
641 struct rvu_block *block;
642 struct rvu_pfvf *pfvf;
643
644 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
645 if (blkaddr < 0)
646 return false;
647
648 /* Registers that can be accessed from PF/VF */
649 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) ||
650 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) {
651 if (offset & 7)
652 return false;
653
654 lf = (offset & 0xFFF) >> 3;
655 block = &rvu->hw->block[blkaddr];
656 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
657 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
658 if (lf >= num_lfs)
659 /* Slot is not valid for that PF/VF */
660 return false;
661
662 /* Translate local LF used by VFs to global CPT LF */
663 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr],
664 req->hdr.pcifunc, lf);
665 if (lf < 0)
666 return false;
667
668 /* Translate local LF's offset to global CPT LF's offset to
669 * access LFX register.
670 */
671 *reg_offset = (req->reg_offset & 0xFF000) + (lf << 3);
672
673 return true;
674 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) {
675 /* Registers that can be accessed from PF */
676 switch (offset) {
677 case CPT_AF_DIAG:
678 case CPT_AF_CTL:
679 case CPT_AF_PF_FUNC:
680 case CPT_AF_BLK_RST:
681 case CPT_AF_CONSTANTS1:
682 case CPT_AF_CTX_FLUSH_TIMER:
683 return true;
684 }
685
686 switch (offset & 0xFF000) {
687 case CPT_AF_EXEX_STS(0):
688 case CPT_AF_EXEX_CTL(0):
689 case CPT_AF_EXEX_CTL2(0):
690 case CPT_AF_EXEX_UCODE_BASE(0):
691 if (offset & 7)
692 return false;
693 break;
694 default:
695 return false;
696 }
697 return true;
698 }
699 return false;
700 }
701
rvu_mbox_handler_cpt_rd_wr_register(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req,struct cpt_rd_wr_reg_msg * rsp)702 int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu,
703 struct cpt_rd_wr_reg_msg *req,
704 struct cpt_rd_wr_reg_msg *rsp)
705 {
706 u64 offset = req->reg_offset;
707 int blkaddr;
708
709 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
710 if (blkaddr < 0)
711 return blkaddr;
712
713 /* This message is accepted only if sent from CPT PF/VF */
714 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
715 !is_cpt_vf(rvu, req->hdr.pcifunc))
716 return CPT_AF_ERR_ACCESS_DENIED;
717
718 if (!validate_and_update_reg_offset(rvu, req, &offset))
719 return CPT_AF_ERR_ACCESS_DENIED;
720
721 rsp->reg_offset = req->reg_offset;
722 rsp->ret_val = req->ret_val;
723 rsp->is_write = req->is_write;
724
725 if (req->is_write)
726 rvu_write64(rvu, blkaddr, offset, req->val);
727 else
728 rsp->val = rvu_read64(rvu, blkaddr, offset);
729
730 return 0;
731 }
732
get_ctx_pc(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)733 static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
734 {
735 if (is_rvu_otx2(rvu))
736 return;
737
738 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC);
739 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC);
740 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC);
741 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr,
742 CPT_AF_CTX_AOP_LATENCY_PC);
743 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC);
744 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr,
745 CPT_AF_CTX_IFETCH_LATENCY_PC);
746 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
747 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr,
748 CPT_AF_CTX_FFETCH_LATENCY_PC);
749 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
750 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr,
751 CPT_AF_CTX_FFETCH_LATENCY_PC);
752 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
753 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr,
754 CPT_AF_CTX_FFETCH_LATENCY_PC);
755 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR);
756 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID);
757 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER);
758
759 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME);
760 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
761 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
762 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
763 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
764 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0));
765 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1));
766 }
767
get_eng_sts(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)768 static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
769 {
770 u16 max_ses, max_ies, max_aes;
771 u32 e_min = 0, e_max = 0;
772 u64 reg;
773
774 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1);
775 max_ses = reg & 0xffff;
776 max_ies = (reg >> 16) & 0xffff;
777 max_aes = (reg >> 32) & 0xffff;
778
779 /* Get AE status */
780 e_min = max_ses + max_ies;
781 e_max = max_ses + max_ies + max_aes;
782 cpt_get_eng_sts(e_min, e_max, rsp, ae);
783 /* Get SE status */
784 e_min = 0;
785 e_max = max_ses;
786 cpt_get_eng_sts(e_min, e_max, rsp, se);
787 /* Get IE status */
788 e_min = max_ses;
789 e_max = max_ses + max_ies;
790 cpt_get_eng_sts(e_min, e_max, rsp, ie);
791 }
792
rvu_mbox_handler_cpt_sts(struct rvu * rvu,struct cpt_sts_req * req,struct cpt_sts_rsp * rsp)793 int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req,
794 struct cpt_sts_rsp *rsp)
795 {
796 int blkaddr;
797
798 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
799 if (blkaddr < 0)
800 return blkaddr;
801
802 /* This message is accepted only if sent from CPT PF/VF */
803 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
804 !is_cpt_vf(rvu, req->hdr.pcifunc))
805 return CPT_AF_ERR_ACCESS_DENIED;
806
807 get_ctx_pc(rvu, rsp, blkaddr);
808
809 /* Get CPT engines status */
810 get_eng_sts(rvu, rsp, blkaddr);
811
812 /* Read CPT instruction PC registers */
813 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC);
814 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC);
815 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC);
816 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC);
817 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC);
818 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr,
819 CPT_AF_ACTIVE_CYCLES_PC);
820 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO);
821 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT);
822 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG);
823
824 return 0;
825 }
826
827 #define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48)
828 #define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32)
829 #define RXC_ACTIVE_THRES GENMASK_ULL(27, 16)
830 #define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0)
831 #define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48)
832 #define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48)
833
cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,int blkaddr,struct cpt_rxc_time_cfg_req * save)834 static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req,
835 int blkaddr, struct cpt_rxc_time_cfg_req *save)
836 {
837 u64 dfrg_reg;
838
839 if (save) {
840 /* Save older config */
841 dfrg_reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
842 save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg);
843 save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg);
844 save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg);
845 save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg);
846
847 save->step = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
848 }
849
850 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres);
851 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit);
852 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres);
853 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit);
854
855 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step);
856 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg);
857 }
858
rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,struct msg_rsp * rsp)859 int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu,
860 struct cpt_rxc_time_cfg_req *req,
861 struct msg_rsp *rsp)
862 {
863 int blkaddr;
864
865 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
866 if (blkaddr < 0)
867 return blkaddr;
868
869 /* This message is accepted only if sent from CPT PF/VF */
870 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
871 !is_cpt_vf(rvu, req->hdr.pcifunc))
872 return CPT_AF_ERR_ACCESS_DENIED;
873
874 cpt_rxc_time_cfg(rvu, req, blkaddr, NULL);
875
876 return 0;
877 }
878
rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)879 int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req,
880 struct msg_rsp *rsp)
881 {
882 return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc);
883 }
884
rvu_mbox_handler_cpt_lf_reset(struct rvu * rvu,struct cpt_lf_rst_req * req,struct msg_rsp * rsp)885 int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req,
886 struct msg_rsp *rsp)
887 {
888 u16 pcifunc = req->hdr.pcifunc;
889 struct rvu_block *block;
890 int cptlf, blkaddr, ret;
891 u16 actual_slot;
892 u64 ctl, ctl2;
893
894 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
895 req->slot, &actual_slot);
896 if (blkaddr < 0)
897 return CPT_AF_ERR_LF_INVALID;
898
899 block = &rvu->hw->block[blkaddr];
900
901 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
902 if (cptlf < 0)
903 return CPT_AF_ERR_LF_INVALID;
904 ctl = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
905 ctl2 = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
906
907 ret = rvu_lf_reset(rvu, block, cptlf);
908 if (ret)
909 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
910 block->addr, cptlf);
911
912 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), ctl);
913 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), ctl2);
914
915 return 0;
916 }
917
rvu_mbox_handler_cpt_flt_eng_info(struct rvu * rvu,struct cpt_flt_eng_info_req * req,struct cpt_flt_eng_info_rsp * rsp)918 int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req,
919 struct cpt_flt_eng_info_rsp *rsp)
920 {
921 struct rvu_block *block;
922 unsigned long flags;
923 int blkaddr, vec;
924
925 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
926 if (blkaddr < 0)
927 return blkaddr;
928
929 block = &rvu->hw->block[blkaddr];
930 for (vec = 0; vec < CPT_10K_AF_INT_VEC_RVU; vec++) {
931 spin_lock_irqsave(&rvu->cpt_intr_lock, flags);
932 rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec];
933 rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec];
934 if (req->reset) {
935 block->cpt_flt_eng_map[vec] = 0x0;
936 block->cpt_rcvrd_eng_map[vec] = 0x0;
937 }
938 spin_unlock_irqrestore(&rvu->cpt_intr_lock, flags);
939 }
940 return 0;
941 }
942
cpt_rxc_teardown(struct rvu * rvu,int blkaddr)943 static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr)
944 {
945 struct cpt_rxc_time_cfg_req req, prev;
946 int timeout = 2000;
947 u64 reg;
948
949 if (is_rvu_otx2(rvu))
950 return;
951
952 /* Set time limit to minimum values, so that rxc entries will be
953 * flushed out quickly.
954 */
955 req.step = 1;
956 req.zombie_thres = 1;
957 req.zombie_limit = 1;
958 req.active_thres = 1;
959 req.active_limit = 1;
960
961 cpt_rxc_time_cfg(rvu, &req, blkaddr, &prev);
962
963 do {
964 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
965 udelay(1);
966 if (FIELD_GET(RXC_ACTIVE_COUNT, reg))
967 timeout--;
968 else
969 break;
970 } while (timeout);
971
972 if (timeout == 0)
973 dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n");
974
975 timeout = 2000;
976 do {
977 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
978 udelay(1);
979 if (FIELD_GET(RXC_ZOMBIE_COUNT, reg))
980 timeout--;
981 else
982 break;
983 } while (timeout);
984
985 if (timeout == 0)
986 dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n");
987
988 /* Restore config */
989 cpt_rxc_time_cfg(rvu, &prev, blkaddr, NULL);
990 }
991
992 #define INFLIGHT GENMASK_ULL(8, 0)
993 #define GRB_CNT GENMASK_ULL(39, 32)
994 #define GWB_CNT GENMASK_ULL(47, 40)
995 #define XQ_XOR GENMASK_ULL(63, 63)
996 #define DQPTR GENMASK_ULL(19, 0)
997 #define NQPTR GENMASK_ULL(51, 32)
998
cpt_lf_disable_iqueue(struct rvu * rvu,int blkaddr,int slot)999 static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot)
1000 {
1001 int timeout = 1000000;
1002 u64 inprog, inst_ptr;
1003 u64 qsize, pending;
1004 int i = 0;
1005
1006 /* Disable instructions enqueuing */
1007 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0);
1008
1009 inprog = rvu_read64(rvu, blkaddr,
1010 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1011 inprog |= BIT_ULL(16);
1012 rvu_write64(rvu, blkaddr,
1013 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog);
1014
1015 qsize = rvu_read64(rvu, blkaddr,
1016 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF;
1017 do {
1018 inst_ptr = rvu_read64(rvu, blkaddr,
1019 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR));
1020 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +
1021 FIELD_GET(NQPTR, inst_ptr) -
1022 FIELD_GET(DQPTR, inst_ptr);
1023 udelay(1);
1024 timeout--;
1025 } while ((pending != 0) && (timeout != 0));
1026
1027 if (timeout == 0)
1028 dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n");
1029
1030 timeout = 1000000;
1031 /* Wait for CPT queue to become execution-quiescent */
1032 do {
1033 inprog = rvu_read64(rvu, blkaddr,
1034 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1035
1036 if ((FIELD_GET(INFLIGHT, inprog) == 0) &&
1037 (FIELD_GET(GRB_CNT, inprog) == 0)) {
1038 i++;
1039 } else {
1040 i = 0;
1041 timeout--;
1042 }
1043 } while ((timeout != 0) && (i < 10));
1044
1045 if (timeout == 0)
1046 dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n");
1047 /* Wait for 2 us to flush all queue writes to memory */
1048 udelay(2);
1049 }
1050
rvu_cpt_lf_teardown(struct rvu * rvu,u16 pcifunc,int blkaddr,int lf,int slot)1051 int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot)
1052 {
1053 u64 reg;
1054
1055 if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc))
1056 cpt_rxc_teardown(rvu, blkaddr);
1057
1058 mutex_lock(&rvu->alias_lock);
1059 /* Enable BAR2 ALIAS for this pcifunc. */
1060 reg = BIT_ULL(16) | pcifunc;
1061 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1062
1063 cpt_lf_disable_iqueue(rvu, blkaddr, slot);
1064
1065 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1066 mutex_unlock(&rvu->alias_lock);
1067
1068 return 0;
1069 }
1070
1071 #define CPT_RES_LEN 16
1072 #define CPT_SE_IE_EGRP 1ULL
1073
cpt_inline_inb_lf_cmd_send(struct rvu * rvu,int blkaddr,int nix_blkaddr)1074 static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr,
1075 int nix_blkaddr)
1076 {
1077 int cpt_pf_num = rvu->cpt_pf_num;
1078 struct cpt_inst_lmtst_req *req;
1079 dma_addr_t res_daddr;
1080 int timeout = 3000;
1081 u8 cpt_idx;
1082 u64 *inst;
1083 u16 *res;
1084 int rc;
1085
1086 res = kzalloc(CPT_RES_LEN, GFP_KERNEL);
1087 if (!res)
1088 return -ENOMEM;
1089
1090 res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN,
1091 DMA_BIDIRECTIONAL);
1092 if (dma_mapping_error(rvu->dev, res_daddr)) {
1093 dev_err(rvu->dev, "DMA mapping failed for CPT result\n");
1094 rc = -EFAULT;
1095 goto res_free;
1096 }
1097 *res = 0xFFFF;
1098
1099 /* Send mbox message to CPT PF */
1100 req = (struct cpt_inst_lmtst_req *)
1101 otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up,
1102 cpt_pf_num, sizeof(*req),
1103 sizeof(struct msg_rsp));
1104 if (!req) {
1105 rc = -ENOMEM;
1106 goto res_daddr_unmap;
1107 }
1108 req->hdr.sig = OTX2_MBOX_REQ_SIG;
1109 req->hdr.id = MBOX_MSG_CPT_INST_LMTST;
1110
1111 inst = req->inst;
1112 /* Prepare CPT_INST_S */
1113 inst[0] = 0;
1114 inst[1] = res_daddr;
1115 /* AF PF FUNC */
1116 inst[2] = 0;
1117 /* Set QORD */
1118 inst[3] = 1;
1119 inst[4] = 0;
1120 inst[5] = 0;
1121 inst[6] = 0;
1122 /* Set EGRP */
1123 inst[7] = CPT_SE_IE_EGRP << 61;
1124
1125 /* Subtract 1 from the NIX-CPT credit count to preserve
1126 * credit counts.
1127 */
1128 cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1;
1129 rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),
1130 BIT_ULL(22) - 1);
1131
1132 otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1133 rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1134 if (rc)
1135 dev_warn(rvu->dev, "notification to pf %d failed\n",
1136 cpt_pf_num);
1137 /* Wait for CPT instruction to be completed */
1138 do {
1139 mdelay(1);
1140 if (*res == 0xFFFF)
1141 timeout--;
1142 else
1143 break;
1144 } while (timeout);
1145
1146 if (timeout == 0)
1147 dev_warn(rvu->dev, "Poll for result hits hard loop counter\n");
1148
1149 res_daddr_unmap:
1150 dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL);
1151 res_free:
1152 kfree(res);
1153
1154 return 0;
1155 }
1156
1157 #define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46)
1158 #define CTX_CAM_CPTR GENMASK_ULL(45, 0)
1159
rvu_cpt_ctx_flush(struct rvu * rvu,u16 pcifunc)1160 int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc)
1161 {
1162 int nix_blkaddr, blkaddr;
1163 u16 max_ctx_entries, i;
1164 int slot = 0, num_lfs;
1165 u64 reg, cam_data;
1166 int rc;
1167
1168 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1169 if (nix_blkaddr < 0)
1170 return -EINVAL;
1171
1172 if (is_rvu_otx2(rvu))
1173 return 0;
1174
1175 blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0;
1176
1177 /* Submit CPT_INST_S to track when all packets have been
1178 * flushed through for the NIX PF FUNC in inline inbound case.
1179 */
1180 rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr);
1181 if (rc)
1182 return rc;
1183
1184 /* Wait for rxc entries to be flushed out */
1185 cpt_rxc_teardown(rvu, blkaddr);
1186
1187 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);
1188 max_ctx_entries = (reg >> 48) & 0xFFF;
1189
1190 mutex_lock(&rvu->rsrc_lock);
1191
1192 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1193 blkaddr);
1194 if (num_lfs == 0) {
1195 dev_warn(rvu->dev, "CPT LF is not configured\n");
1196 goto unlock;
1197 }
1198
1199 /* Enable BAR2 ALIAS for this pcifunc. */
1200 reg = BIT_ULL(16) | pcifunc;
1201 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1202
1203 for (i = 0; i < max_ctx_entries; i++) {
1204 cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i));
1205
1206 if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) &&
1207 FIELD_GET(CTX_CAM_CPTR, cam_data)) {
1208 reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data);
1209 rvu_write64(rvu, blkaddr,
1210 CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH),
1211 reg);
1212 }
1213 }
1214 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1215
1216 unlock:
1217 mutex_unlock(&rvu->rsrc_lock);
1218
1219 return 0;
1220 }
1221
rvu_cpt_init(struct rvu * rvu)1222 int rvu_cpt_init(struct rvu *rvu)
1223 {
1224 /* Retrieve CPT PF number */
1225 rvu->cpt_pf_num = get_cpt_pf_num(rvu);
1226 spin_lock_init(&rvu->cpt_intr_lock);
1227
1228 return 0;
1229 }
1230