1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3  *
4  * Copyright (C) 2021 Marvell.
5  *
6  */
7 
8 #include "cn10k.h"
9 #include "otx2_reg.h"
10 #include "otx2_struct.h"
11 
12 static struct dev_hw_ops	otx2_hw_ops = {
13 	.sq_aq_init = otx2_sq_aq_init,
14 	.sqe_flush = otx2_sqe_flush,
15 	.aura_freeptr = otx2_aura_freeptr,
16 	.refill_pool_ptrs = otx2_refill_pool_ptrs,
17 };
18 
19 static struct dev_hw_ops cn10k_hw_ops = {
20 	.sq_aq_init = cn10k_sq_aq_init,
21 	.sqe_flush = cn10k_sqe_flush,
22 	.aura_freeptr = cn10k_aura_freeptr,
23 	.refill_pool_ptrs = cn10k_refill_pool_ptrs,
24 };
25 
26 int cn10k_lmtst_init(struct otx2_nic *pfvf)
27 {
28 
29 	struct lmtst_tbl_setup_req *req;
30 	struct otx2_lmt_info *lmt_info;
31 	int err, cpu;
32 
33 	if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
34 		pfvf->hw_ops = &otx2_hw_ops;
35 		return 0;
36 	}
37 
38 	pfvf->hw_ops = &cn10k_hw_ops;
39 	/* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/
40 	pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);
41 	pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);
42 
43 	mutex_lock(&pfvf->mbox.lock);
44 	req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);
45 	if (!req) {
46 		mutex_unlock(&pfvf->mbox.lock);
47 		return -ENOMEM;
48 	}
49 
50 	req->use_local_lmt_region = true;
51 
52 	err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,
53 			 LMT_LINE_SIZE);
54 	if (err) {
55 		mutex_unlock(&pfvf->mbox.lock);
56 		return err;
57 	}
58 	pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;
59 	req->lmt_iova = (u64)pfvf->dync_lmt->iova;
60 
61 	err = otx2_sync_mbox_msg(&pfvf->mbox);
62 	mutex_unlock(&pfvf->mbox.lock);
63 
64 	for_each_possible_cpu(cpu) {
65 		lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);
66 		lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +
67 				      (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));
68 		lmt_info->lmt_id = cpu * LMT_BURST_SIZE;
69 	}
70 
71 	return 0;
72 }
73 EXPORT_SYMBOL(cn10k_lmtst_init);
74 
75 int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura)
76 {
77 	struct nix_cn10k_aq_enq_req *aq;
78 	struct otx2_nic *pfvf = dev;
79 
80 	/* Get memory to put this msg */
81 	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
82 	if (!aq)
83 		return -ENOMEM;
84 
85 	aq->sq.cq = pfvf->hw.rx_queues + qidx;
86 	aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
87 	aq->sq.cq_ena = 1;
88 	aq->sq.ena = 1;
89 	/* Only one SMQ is allocated, map all SQ's to that SMQ  */
90 	aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0];
91 	aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->max_frs);
92 	aq->sq.default_chan = pfvf->hw.tx_chan_base;
93 	aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
94 	aq->sq.sqb_aura = sqb_aura;
95 	aq->sq.sq_int_ena = NIX_SQINT_BITS;
96 	aq->sq.qint_idx = 0;
97 	/* Due pipelining impact minimum 2000 unused SQ CQE's
98 	 * need to maintain to avoid CQ overflow.
99 	 */
100 	aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));
101 
102 	/* Fill AQ info */
103 	aq->qidx = qidx;
104 	aq->ctype = NIX_AQ_CTYPE_SQ;
105 	aq->op = NIX_AQ_INSTOP_INIT;
106 
107 	return otx2_sync_mbox_msg(&pfvf->mbox);
108 }
109 
110 #define NPA_MAX_BURST 16
111 void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
112 {
113 	struct otx2_nic *pfvf = dev;
114 	u64 ptrs[NPA_MAX_BURST];
115 	int num_ptrs = 1;
116 	dma_addr_t bufptr;
117 
118 	/* Refill pool with new buffers */
119 	while (cq->pool_ptrs) {
120 		if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
121 			if (num_ptrs--)
122 				__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
123 						     num_ptrs);
124 			break;
125 		}
126 		cq->pool_ptrs--;
127 		ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;
128 		num_ptrs++;
129 		if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
130 			__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
131 					     num_ptrs);
132 			num_ptrs = 1;
133 		}
134 	}
135 }
136 
137 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)
138 {
139 	struct otx2_lmt_info *lmt_info;
140 	struct otx2_nic *pfvf = dev;
141 	u64 val = 0, tar_addr = 0;
142 
143 	lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
144 	/* FIXME: val[0:10] LMT_ID.
145 	 * [12:15] no of LMTST - 1 in the burst.
146 	 * [19:63] data size of each LMTST in the burst except first.
147 	 */
148 	val = (lmt_info->lmt_id & 0x7FF);
149 	/* Target address for LMTST flush tells HW how many 128bit
150 	 * words are present.
151 	 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
152 	 */
153 	tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;
154 	dma_wmb();
155 	memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);
156 	cn10k_lmt_flush(val, tar_addr);
157 
158 	sq->head++;
159 	sq->head &= (sq->sqe_cnt - 1);
160 }
161 
162 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)
163 {
164 	struct nix_bandprof_free_req *req;
165 	int rc;
166 
167 	if (is_dev_otx2(pfvf->pdev))
168 		return 0;
169 
170 	mutex_lock(&pfvf->mbox.lock);
171 
172 	req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
173 	if (!req) {
174 		rc =  -ENOMEM;
175 		goto out;
176 	}
177 
178 	/* Free all bandwidth profiles allocated */
179 	req->free_all = true;
180 
181 	rc = otx2_sync_mbox_msg(&pfvf->mbox);
182 out:
183 	mutex_unlock(&pfvf->mbox.lock);
184 	return rc;
185 }
186 
187 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)
188 {
189 	struct nix_bandprof_alloc_req *req;
190 	struct nix_bandprof_alloc_rsp *rsp;
191 	int rc;
192 
193 	req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);
194 	if (!req)
195 		return  -ENOMEM;
196 
197 	req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
198 
199 	rc = otx2_sync_mbox_msg(&pfvf->mbox);
200 	if (rc)
201 		goto out;
202 
203 	rsp = (struct  nix_bandprof_alloc_rsp *)
204 	       otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
205 	if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {
206 		rc = -EIO;
207 		goto out;
208 	}
209 
210 	*leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];
211 out:
212 	if (rc) {
213 		dev_warn(pfvf->dev,
214 			 "Failed to allocate ingress bandwidth policer\n");
215 	}
216 
217 	return rc;
218 }
219 
220 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)
221 {
222 	struct otx2_hw *hw = &pfvf->hw;
223 	int ret;
224 
225 	mutex_lock(&pfvf->mbox.lock);
226 
227 	ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);
228 
229 	mutex_unlock(&pfvf->mbox.lock);
230 
231 	return ret;
232 }
233 
234 #define POLICER_TIMESTAMP	  1  /* 1 second */
235 #define MAX_RATE_EXP		  22 /* Valid rate exponent range: 0 - 22 */
236 
237 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,
238 					u32 *burst_mantissa)
239 {
240 	int tmp;
241 
242 	/* Burst is calculated as
243 	 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]
244 	 * This is the upper limit on number tokens (bytes) that
245 	 * can be accumulated in the bucket.
246 	 */
247 	*burst_exp = ilog2(burst);
248 	if (burst < 256) {
249 		/* No float: can't express mantissa in this case */
250 		*burst_mantissa = 0;
251 		return;
252 	}
253 
254 	if (*burst_exp > MAX_RATE_EXP)
255 		*burst_exp = MAX_RATE_EXP;
256 
257 	/* Calculate mantissa
258 	 * Find remaining bytes 'burst - 2^burst_exp'
259 	 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)
260 	 */
261 	tmp = burst - rounddown_pow_of_two(burst);
262 	*burst_mantissa = tmp / (1UL << (*burst_exp - 8));
263 }
264 
265 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
266 				       u32 *rate_mantissa, u32 *rdiv)
267 {
268 	u32 div = 0;
269 	u32 exp = 0;
270 	u64 tmp;
271 
272 	/* Figure out mantissa, exponent and divider from given max pkt rate
273 	 *
274 	 * To achieve desired rate HW adds
275 	 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every
276 	 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
277 	 * Here policer timeunit is 2 usecs and rate is in bits per sec.
278 	 * Since floating point cannot be used below algorithm uses 1000000
279 	 * scale factor to support rates upto 100Gbps.
280 	 */
281 	tmp = rate * 32 * 2;
282 	if (tmp < 256000000) {
283 		while (tmp < 256000000) {
284 			tmp = tmp * 2;
285 			div++;
286 		}
287 	} else {
288 		for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)
289 			tmp = tmp / 2;
290 
291 		if (exp > MAX_RATE_EXP)
292 			exp = MAX_RATE_EXP;
293 	}
294 
295 	*rate_mantissa = (tmp - 256000000) / 1000000;
296 	*rate_exp = exp;
297 	*rdiv = div;
298 }
299 
300 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,
301 			       u16 policer, bool map)
302 {
303 	struct nix_cn10k_aq_enq_req *aq;
304 
305 	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
306 	if (!aq)
307 		return -ENOMEM;
308 
309 	/* Enable policing and set the bandwidth profile (policer) index */
310 	if (map)
311 		aq->rq.policer_ena = 1;
312 	else
313 		aq->rq.policer_ena = 0;
314 	aq->rq_mask.policer_ena = 1;
315 
316 	aq->rq.band_prof_id = policer;
317 	aq->rq_mask.band_prof_id = GENMASK(9, 0);
318 
319 	/* Fill AQ info */
320 	aq->qidx = rq_idx;
321 	aq->ctype = NIX_AQ_CTYPE_RQ;
322 	aq->op = NIX_AQ_INSTOP_WRITE;
323 
324 	return otx2_sync_mbox_msg(&pfvf->mbox);
325 }
326 
327 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)
328 {
329 	struct nix_bandprof_free_req *req;
330 
331 	req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
332 	if (!req)
333 		return -ENOMEM;
334 
335 	req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
336 	req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;
337 
338 	return otx2_sync_mbox_msg(&pfvf->mbox);
339 }
340 
341 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)
342 {
343 	struct otx2_hw *hw = &pfvf->hw;
344 	int qidx, rc;
345 
346 	mutex_lock(&pfvf->mbox.lock);
347 
348 	/* Remove RQ's policer mapping */
349 	for (qidx = 0; qidx < hw->rx_queues; qidx++)
350 		cn10k_map_unmap_rq_policer(pfvf, qidx,
351 					   hw->matchall_ipolicer, false);
352 
353 	rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);
354 
355 	mutex_unlock(&pfvf->mbox.lock);
356 	return rc;
357 }
358 
359 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,
360 			    u32 burst, u64 rate, bool pps)
361 {
362 	struct nix_cn10k_aq_enq_req *aq;
363 	u32 burst_exp, burst_mantissa;
364 	u32 rate_exp, rate_mantissa;
365 	u32 rdiv;
366 
367 	/* Get exponent and mantissa values for the desired rate */
368 	cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);
369 	cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);
370 
371 	/* Init bandwidth profile */
372 	aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
373 	if (!aq)
374 		return -ENOMEM;
375 
376 	/* Set initial color mode to blind */
377 	aq->prof.icolor = 0x03;
378 	aq->prof_mask.icolor = 0x03;
379 
380 	/* Set rate and burst values */
381 	aq->prof.cir_exponent = rate_exp;
382 	aq->prof_mask.cir_exponent = 0x1F;
383 
384 	aq->prof.cir_mantissa = rate_mantissa;
385 	aq->prof_mask.cir_mantissa = 0xFF;
386 
387 	aq->prof.cbs_exponent = burst_exp;
388 	aq->prof_mask.cbs_exponent = 0x1F;
389 
390 	aq->prof.cbs_mantissa = burst_mantissa;
391 	aq->prof_mask.cbs_mantissa = 0xFF;
392 
393 	aq->prof.rdiv = rdiv;
394 	aq->prof_mask.rdiv = 0xF;
395 
396 	if (pps) {
397 		/* The amount of decremented tokens is calculated according to
398 		 * the following equation:
399 		 * max([ LMODE ? 0 : (packet_length - LXPTR)] +
400 		 *	     ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],
401 		 *	1/256)
402 		 * if LMODE is 1 then rate limiting will be based on
403 		 * PPS otherwise bps.
404 		 * The aim of the ADJUST value is to specify a token cost per
405 		 * packet in contrary to the packet length that specifies a
406 		 * cost per byte. To rate limit based on PPS adjust mantissa
407 		 * is set as 384 and exponent as 1 so that number of tokens
408 		 * decremented becomes 1 i.e, 1 token per packeet.
409 		 */
410 		aq->prof.adjust_exponent = 1;
411 		aq->prof_mask.adjust_exponent = 0x1F;
412 
413 		aq->prof.adjust_mantissa = 384;
414 		aq->prof_mask.adjust_mantissa = 0x1FF;
415 
416 		aq->prof.lmode = 0x1;
417 		aq->prof_mask.lmode = 0x1;
418 	}
419 
420 	/* Two rate three color marker
421 	 * With PEIR/EIR set to zero, color will be either green or red
422 	 */
423 	aq->prof.meter_algo = 2;
424 	aq->prof_mask.meter_algo = 0x3;
425 
426 	aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;
427 	aq->prof_mask.rc_action = 0x3;
428 
429 	aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
430 	aq->prof_mask.yc_action = 0x3;
431 
432 	aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
433 	aq->prof_mask.gc_action = 0x3;
434 
435 	/* Setting exponent value as 24 and mantissa as 0 configures
436 	 * the bucket with zero values making bucket unused. Peak
437 	 * information rate and Excess information rate buckets are
438 	 * unused here.
439 	 */
440 	aq->prof.peir_exponent = 24;
441 	aq->prof_mask.peir_exponent = 0x1F;
442 
443 	aq->prof.peir_mantissa = 0;
444 	aq->prof_mask.peir_mantissa = 0xFF;
445 
446 	aq->prof.pebs_exponent = 24;
447 	aq->prof_mask.pebs_exponent = 0x1F;
448 
449 	aq->prof.pebs_mantissa = 0;
450 	aq->prof_mask.pebs_mantissa = 0xFF;
451 
452 	/* Fill AQ info */
453 	aq->qidx = profile;
454 	aq->ctype = NIX_AQ_CTYPE_BANDPROF;
455 	aq->op = NIX_AQ_INSTOP_WRITE;
456 
457 	return otx2_sync_mbox_msg(&pfvf->mbox);
458 }
459 
460 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,
461 				     u32 burst, u64 rate)
462 {
463 	struct otx2_hw *hw = &pfvf->hw;
464 	int qidx, rc;
465 
466 	mutex_lock(&pfvf->mbox.lock);
467 
468 	rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,
469 				     rate, false);
470 	if (rc)
471 		goto out;
472 
473 	for (qidx = 0; qidx < hw->rx_queues; qidx++) {
474 		rc = cn10k_map_unmap_rq_policer(pfvf, qidx,
475 						hw->matchall_ipolicer, true);
476 		if (rc)
477 			break;
478 	}
479 
480 out:
481 	mutex_unlock(&pfvf->mbox.lock);
482 	return rc;
483 }
484