1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2021 Marvell.
5 *
6 */
7
8 #include "cn10k.h"
9 #include "otx2_reg.h"
10 #include "otx2_struct.h"
11
12 static struct dev_hw_ops otx2_hw_ops = {
13 .sq_aq_init = otx2_sq_aq_init,
14 .sqe_flush = otx2_sqe_flush,
15 .aura_freeptr = otx2_aura_freeptr,
16 .refill_pool_ptrs = otx2_refill_pool_ptrs,
17 };
18
19 static struct dev_hw_ops cn10k_hw_ops = {
20 .sq_aq_init = cn10k_sq_aq_init,
21 .sqe_flush = cn10k_sqe_flush,
22 .aura_freeptr = cn10k_aura_freeptr,
23 .refill_pool_ptrs = cn10k_refill_pool_ptrs,
24 };
25
cn10k_lmtst_init(struct otx2_nic * pfvf)26 int cn10k_lmtst_init(struct otx2_nic *pfvf)
27 {
28
29 struct lmtst_tbl_setup_req *req;
30 struct otx2_lmt_info *lmt_info;
31 int err, cpu;
32
33 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
34 pfvf->hw_ops = &otx2_hw_ops;
35 return 0;
36 }
37
38 pfvf->hw_ops = &cn10k_hw_ops;
39 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/
40 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);
41 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);
42
43 mutex_lock(&pfvf->mbox.lock);
44 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);
45 if (!req) {
46 mutex_unlock(&pfvf->mbox.lock);
47 return -ENOMEM;
48 }
49
50 req->use_local_lmt_region = true;
51
52 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,
53 LMT_LINE_SIZE);
54 if (err) {
55 mutex_unlock(&pfvf->mbox.lock);
56 return err;
57 }
58 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;
59 req->lmt_iova = (u64)pfvf->dync_lmt->iova;
60
61 err = otx2_sync_mbox_msg(&pfvf->mbox);
62 mutex_unlock(&pfvf->mbox.lock);
63
64 for_each_possible_cpu(cpu) {
65 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);
66 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +
67 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));
68 lmt_info->lmt_id = cpu * LMT_BURST_SIZE;
69 }
70
71 return 0;
72 }
73 EXPORT_SYMBOL(cn10k_lmtst_init);
74
cn10k_sq_aq_init(void * dev,u16 qidx,u16 sqb_aura)75 int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura)
76 {
77 struct nix_cn10k_aq_enq_req *aq;
78 struct otx2_nic *pfvf = dev;
79
80 /* Get memory to put this msg */
81 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
82 if (!aq)
83 return -ENOMEM;
84
85 aq->sq.cq = pfvf->hw.rx_queues + qidx;
86 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
87 aq->sq.cq_ena = 1;
88 aq->sq.ena = 1;
89 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
90 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
91 aq->sq.default_chan = pfvf->hw.tx_chan_base;
92 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
93 aq->sq.sqb_aura = sqb_aura;
94 aq->sq.sq_int_ena = NIX_SQINT_BITS;
95 aq->sq.qint_idx = 0;
96 /* Due pipelining impact minimum 2000 unused SQ CQE's
97 * need to maintain to avoid CQ overflow.
98 */
99 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));
100
101 /* Fill AQ info */
102 aq->qidx = qidx;
103 aq->ctype = NIX_AQ_CTYPE_SQ;
104 aq->op = NIX_AQ_INSTOP_INIT;
105
106 return otx2_sync_mbox_msg(&pfvf->mbox);
107 }
108
109 #define NPA_MAX_BURST 16
cn10k_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)110 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
111 {
112 struct otx2_nic *pfvf = dev;
113 int cnt = cq->pool_ptrs;
114 u64 ptrs[NPA_MAX_BURST];
115 dma_addr_t bufptr;
116 int num_ptrs = 1;
117
118 /* Refill pool with new buffers */
119 while (cq->pool_ptrs) {
120 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
121 if (num_ptrs--)
122 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
123 num_ptrs);
124 break;
125 }
126 cq->pool_ptrs--;
127 ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;
128 num_ptrs++;
129 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
130 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
131 num_ptrs);
132 num_ptrs = 1;
133 }
134 }
135 return cnt - cq->pool_ptrs;
136 }
137
cn10k_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)138 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)
139 {
140 struct otx2_lmt_info *lmt_info;
141 struct otx2_nic *pfvf = dev;
142 u64 val = 0, tar_addr = 0;
143
144 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
145 /* FIXME: val[0:10] LMT_ID.
146 * [12:15] no of LMTST - 1 in the burst.
147 * [19:63] data size of each LMTST in the burst except first.
148 */
149 val = (lmt_info->lmt_id & 0x7FF);
150 /* Target address for LMTST flush tells HW how many 128bit
151 * words are present.
152 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
153 */
154 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;
155 dma_wmb();
156 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);
157 cn10k_lmt_flush(val, tar_addr);
158
159 sq->head++;
160 sq->head &= (sq->sqe_cnt - 1);
161 }
162
cn10k_free_all_ipolicers(struct otx2_nic * pfvf)163 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)
164 {
165 struct nix_bandprof_free_req *req;
166 int rc;
167
168 if (is_dev_otx2(pfvf->pdev))
169 return 0;
170
171 mutex_lock(&pfvf->mbox.lock);
172
173 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
174 if (!req) {
175 rc = -ENOMEM;
176 goto out;
177 }
178
179 /* Free all bandwidth profiles allocated */
180 req->free_all = true;
181
182 rc = otx2_sync_mbox_msg(&pfvf->mbox);
183 out:
184 mutex_unlock(&pfvf->mbox.lock);
185 return rc;
186 }
187
cn10k_alloc_leaf_profile(struct otx2_nic * pfvf,u16 * leaf)188 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)
189 {
190 struct nix_bandprof_alloc_req *req;
191 struct nix_bandprof_alloc_rsp *rsp;
192 int rc;
193
194 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);
195 if (!req)
196 return -ENOMEM;
197
198 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
199
200 rc = otx2_sync_mbox_msg(&pfvf->mbox);
201 if (rc)
202 goto out;
203
204 rsp = (struct nix_bandprof_alloc_rsp *)
205 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
206 if (IS_ERR(rsp)) {
207 rc = PTR_ERR(rsp);
208 goto out;
209 }
210
211 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {
212 rc = -EIO;
213 goto out;
214 }
215
216 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];
217 out:
218 if (rc) {
219 dev_warn(pfvf->dev,
220 "Failed to allocate ingress bandwidth policer\n");
221 }
222
223 return rc;
224 }
225
cn10k_alloc_matchall_ipolicer(struct otx2_nic * pfvf)226 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)
227 {
228 struct otx2_hw *hw = &pfvf->hw;
229 int ret;
230
231 mutex_lock(&pfvf->mbox.lock);
232
233 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);
234
235 mutex_unlock(&pfvf->mbox.lock);
236
237 return ret;
238 }
239
240 #define POLICER_TIMESTAMP 1 /* 1 second */
241 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */
242
cn10k_get_ingress_burst_cfg(u32 burst,u32 * burst_exp,u32 * burst_mantissa)243 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,
244 u32 *burst_mantissa)
245 {
246 int tmp;
247
248 /* Burst is calculated as
249 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]
250 * This is the upper limit on number tokens (bytes) that
251 * can be accumulated in the bucket.
252 */
253 *burst_exp = ilog2(burst);
254 if (burst < 256) {
255 /* No float: can't express mantissa in this case */
256 *burst_mantissa = 0;
257 return;
258 }
259
260 if (*burst_exp > MAX_RATE_EXP)
261 *burst_exp = MAX_RATE_EXP;
262
263 /* Calculate mantissa
264 * Find remaining bytes 'burst - 2^burst_exp'
265 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)
266 */
267 tmp = burst - rounddown_pow_of_two(burst);
268 *burst_mantissa = tmp / (1UL << (*burst_exp - 8));
269 }
270
cn10k_get_ingress_rate_cfg(u64 rate,u32 * rate_exp,u32 * rate_mantissa,u32 * rdiv)271 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
272 u32 *rate_mantissa, u32 *rdiv)
273 {
274 u32 div = 0;
275 u32 exp = 0;
276 u64 tmp;
277
278 /* Figure out mantissa, exponent and divider from given max pkt rate
279 *
280 * To achieve desired rate HW adds
281 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every
282 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
283 * Here policer timeunit is 2 usecs and rate is in bits per sec.
284 * Since floating point cannot be used below algorithm uses 1000000
285 * scale factor to support rates upto 100Gbps.
286 */
287 tmp = rate * 32 * 2;
288 if (tmp < 256000000) {
289 while (tmp < 256000000) {
290 tmp = tmp * 2;
291 div++;
292 }
293 } else {
294 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)
295 tmp = tmp / 2;
296
297 if (exp > MAX_RATE_EXP)
298 exp = MAX_RATE_EXP;
299 }
300
301 *rate_mantissa = (tmp - 256000000) / 1000000;
302 *rate_exp = exp;
303 *rdiv = div;
304 }
305
cn10k_map_unmap_rq_policer(struct otx2_nic * pfvf,int rq_idx,u16 policer,bool map)306 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,
307 u16 policer, bool map)
308 {
309 struct nix_cn10k_aq_enq_req *aq;
310
311 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
312 if (!aq)
313 return -ENOMEM;
314
315 /* Enable policing and set the bandwidth profile (policer) index */
316 if (map)
317 aq->rq.policer_ena = 1;
318 else
319 aq->rq.policer_ena = 0;
320 aq->rq_mask.policer_ena = 1;
321
322 aq->rq.band_prof_id = policer;
323 aq->rq_mask.band_prof_id = GENMASK(9, 0);
324
325 /* Fill AQ info */
326 aq->qidx = rq_idx;
327 aq->ctype = NIX_AQ_CTYPE_RQ;
328 aq->op = NIX_AQ_INSTOP_WRITE;
329
330 return otx2_sync_mbox_msg(&pfvf->mbox);
331 }
332
cn10k_free_leaf_profile(struct otx2_nic * pfvf,u16 leaf)333 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)
334 {
335 struct nix_bandprof_free_req *req;
336
337 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
338 if (!req)
339 return -ENOMEM;
340
341 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
342 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;
343
344 return otx2_sync_mbox_msg(&pfvf->mbox);
345 }
346
cn10k_free_matchall_ipolicer(struct otx2_nic * pfvf)347 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)
348 {
349 struct otx2_hw *hw = &pfvf->hw;
350 int qidx, rc;
351
352 mutex_lock(&pfvf->mbox.lock);
353
354 /* Remove RQ's policer mapping */
355 for (qidx = 0; qidx < hw->rx_queues; qidx++)
356 cn10k_map_unmap_rq_policer(pfvf, qidx,
357 hw->matchall_ipolicer, false);
358
359 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);
360
361 mutex_unlock(&pfvf->mbox.lock);
362 return rc;
363 }
364
cn10k_set_ipolicer_rate(struct otx2_nic * pfvf,u16 profile,u32 burst,u64 rate,bool pps)365 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,
366 u32 burst, u64 rate, bool pps)
367 {
368 struct nix_cn10k_aq_enq_req *aq;
369 u32 burst_exp, burst_mantissa;
370 u32 rate_exp, rate_mantissa;
371 u32 rdiv;
372
373 /* Get exponent and mantissa values for the desired rate */
374 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);
375 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);
376
377 /* Init bandwidth profile */
378 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
379 if (!aq)
380 return -ENOMEM;
381
382 /* Set initial color mode to blind */
383 aq->prof.icolor = 0x03;
384 aq->prof_mask.icolor = 0x03;
385
386 /* Set rate and burst values */
387 aq->prof.cir_exponent = rate_exp;
388 aq->prof_mask.cir_exponent = 0x1F;
389
390 aq->prof.cir_mantissa = rate_mantissa;
391 aq->prof_mask.cir_mantissa = 0xFF;
392
393 aq->prof.cbs_exponent = burst_exp;
394 aq->prof_mask.cbs_exponent = 0x1F;
395
396 aq->prof.cbs_mantissa = burst_mantissa;
397 aq->prof_mask.cbs_mantissa = 0xFF;
398
399 aq->prof.rdiv = rdiv;
400 aq->prof_mask.rdiv = 0xF;
401
402 if (pps) {
403 /* The amount of decremented tokens is calculated according to
404 * the following equation:
405 * max([ LMODE ? 0 : (packet_length - LXPTR)] +
406 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],
407 * 1/256)
408 * if LMODE is 1 then rate limiting will be based on
409 * PPS otherwise bps.
410 * The aim of the ADJUST value is to specify a token cost per
411 * packet in contrary to the packet length that specifies a
412 * cost per byte. To rate limit based on PPS adjust mantissa
413 * is set as 384 and exponent as 1 so that number of tokens
414 * decremented becomes 1 i.e, 1 token per packeet.
415 */
416 aq->prof.adjust_exponent = 1;
417 aq->prof_mask.adjust_exponent = 0x1F;
418
419 aq->prof.adjust_mantissa = 384;
420 aq->prof_mask.adjust_mantissa = 0x1FF;
421
422 aq->prof.lmode = 0x1;
423 aq->prof_mask.lmode = 0x1;
424 }
425
426 /* Two rate three color marker
427 * With PEIR/EIR set to zero, color will be either green or red
428 */
429 aq->prof.meter_algo = 2;
430 aq->prof_mask.meter_algo = 0x3;
431
432 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;
433 aq->prof_mask.rc_action = 0x3;
434
435 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
436 aq->prof_mask.yc_action = 0x3;
437
438 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
439 aq->prof_mask.gc_action = 0x3;
440
441 /* Setting exponent value as 24 and mantissa as 0 configures
442 * the bucket with zero values making bucket unused. Peak
443 * information rate and Excess information rate buckets are
444 * unused here.
445 */
446 aq->prof.peir_exponent = 24;
447 aq->prof_mask.peir_exponent = 0x1F;
448
449 aq->prof.peir_mantissa = 0;
450 aq->prof_mask.peir_mantissa = 0xFF;
451
452 aq->prof.pebs_exponent = 24;
453 aq->prof_mask.pebs_exponent = 0x1F;
454
455 aq->prof.pebs_mantissa = 0;
456 aq->prof_mask.pebs_mantissa = 0xFF;
457
458 aq->prof.hl_en = 0;
459 aq->prof_mask.hl_en = 1;
460
461 /* Fill AQ info */
462 aq->qidx = profile;
463 aq->ctype = NIX_AQ_CTYPE_BANDPROF;
464 aq->op = NIX_AQ_INSTOP_WRITE;
465
466 return otx2_sync_mbox_msg(&pfvf->mbox);
467 }
468
cn10k_set_matchall_ipolicer_rate(struct otx2_nic * pfvf,u32 burst,u64 rate)469 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,
470 u32 burst, u64 rate)
471 {
472 struct otx2_hw *hw = &pfvf->hw;
473 int qidx, rc;
474
475 mutex_lock(&pfvf->mbox.lock);
476
477 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,
478 rate, false);
479 if (rc)
480 goto out;
481
482 for (qidx = 0; qidx < hw->rx_queues; qidx++) {
483 rc = cn10k_map_unmap_rq_policer(pfvf, qidx,
484 hw->matchall_ipolicer, true);
485 if (rc)
486 break;
487 }
488
489 out:
490 mutex_unlock(&pfvf->mbox.lock);
491 return rc;
492 }
493