1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2018 Netronome Systems, Inc. */
3 
4 #include <linux/bitops.h>
5 #include <linux/kernel.h>
6 #include <linux/log2.h>
7 
8 #include "../nfpcore/nfp_cpp.h"
9 #include "../nfpcore/nfp_nffw.h"
10 #include "../nfp_app.h"
11 #include "../nfp_abi.h"
12 #include "../nfp_main.h"
13 #include "../nfp_net.h"
14 #include "main.h"
15 
16 #define NFP_NUM_PRIOS_SYM_NAME	"_abi_pci_dscp_num_prio_%u"
17 #define NFP_NUM_BANDS_SYM_NAME	"_abi_pci_dscp_num_band_%u"
18 #define NFP_ACT_MASK_SYM_NAME	"_abi_nfd_out_q_actions_%u"
19 
20 #define NFP_RED_SUPPORT_SYM_NAME	"_abi_nfd_out_red_offload_%u"
21 
22 #define NFP_QLVL_SYM_NAME	"_abi_nfd_out_q_lvls_%u%s"
23 #define NFP_QLVL_STRIDE		16
24 #define NFP_QLVL_BLOG_BYTES	0
25 #define NFP_QLVL_BLOG_PKTS	4
26 #define NFP_QLVL_THRS		8
27 #define NFP_QLVL_ACT		12
28 
29 #define NFP_QMSTAT_SYM_NAME	"_abi_nfdqm%u_stats%s"
30 #define NFP_QMSTAT_STRIDE	32
31 #define NFP_QMSTAT_NON_STO	0
32 #define NFP_QMSTAT_STO		8
33 #define NFP_QMSTAT_DROP		16
34 #define NFP_QMSTAT_ECN		24
35 
36 #define NFP_Q_STAT_SYM_NAME	"_abi_nfd_rxq_stats%u%s"
37 #define NFP_Q_STAT_STRIDE	16
38 #define NFP_Q_STAT_PKTS		0
39 #define NFP_Q_STAT_BYTES	8
40 
41 #define NFP_NET_ABM_MBOX_CMD		NFP_NET_CFG_MBOX_SIMPLE_CMD
42 #define NFP_NET_ABM_MBOX_RET		NFP_NET_CFG_MBOX_SIMPLE_RET
43 #define NFP_NET_ABM_MBOX_DATALEN	NFP_NET_CFG_MBOX_SIMPLE_VAL
44 #define NFP_NET_ABM_MBOX_RESERVED	(NFP_NET_CFG_MBOX_SIMPLE_VAL + 4)
45 #define NFP_NET_ABM_MBOX_DATA		(NFP_NET_CFG_MBOX_SIMPLE_VAL + 8)
46 
47 static int
48 nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym,
49 		  unsigned int stride, unsigned int offset, unsigned int band,
50 		  unsigned int queue, bool is_u64, u64 *res)
51 {
52 	struct nfp_cpp *cpp = alink->abm->app->cpp;
53 	u64 val, sym_offset;
54 	unsigned int qid;
55 	u32 val32;
56 	int err;
57 
58 	qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
59 
60 	sym_offset = qid * stride + offset;
61 	if (is_u64)
62 		err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val);
63 	else
64 		err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32);
65 	if (err) {
66 		nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n",
67 			alink->id, band, queue, alink->queue_base);
68 		return err;
69 	}
70 
71 	*res = is_u64 ? val : val32;
72 	return 0;
73 }
74 
75 int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val)
76 {
77 	struct nfp_cpp *cpp = abm->app->cpp;
78 	u64 sym_offset;
79 	int err;
80 
81 	__clear_bit(id, abm->threshold_undef);
82 	if (abm->thresholds[id] == val)
83 		return 0;
84 
85 	sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS;
86 	err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val);
87 	if (err) {
88 		nfp_err(cpp,
89 			"RED offload setting level failed on subqueue %d\n",
90 			id);
91 		return err;
92 	}
93 
94 	abm->thresholds[id] = val;
95 	return 0;
96 }
97 
98 int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band,
99 			   unsigned int queue, u32 val)
100 {
101 	unsigned int threshold;
102 
103 	threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
104 
105 	return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val);
106 }
107 
108 int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id,
109 			     enum nfp_abm_q_action act)
110 {
111 	struct nfp_cpp *cpp = abm->app->cpp;
112 	u64 sym_offset;
113 	int err;
114 
115 	if (abm->actions[id] == act)
116 		return 0;
117 
118 	sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT;
119 	err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, act);
120 	if (err) {
121 		nfp_err(cpp,
122 			"RED offload setting action failed on subqueue %d\n",
123 			id);
124 		return err;
125 	}
126 
127 	abm->actions[id] = act;
128 	return 0;
129 }
130 
131 int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band,
132 			   unsigned int queue, enum nfp_abm_q_action act)
133 {
134 	unsigned int qid;
135 
136 	qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue;
137 
138 	return __nfp_abm_ctrl_set_q_act(alink->abm, qid, act);
139 }
140 
141 u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue)
142 {
143 	unsigned int band;
144 	u64 val, sum = 0;
145 
146 	for (band = 0; band < alink->abm->num_bands; band++) {
147 		if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
148 				      NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO,
149 				      band, queue, true, &val))
150 			return 0;
151 		sum += val;
152 	}
153 
154 	return sum;
155 }
156 
157 u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue)
158 {
159 	unsigned int band;
160 	u64 val, sum = 0;
161 
162 	for (band = 0; band < alink->abm->num_bands; band++) {
163 		if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
164 				      NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO,
165 				      band, queue, true, &val))
166 			return 0;
167 		sum += val;
168 	}
169 
170 	return sum;
171 }
172 
173 static int
174 nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band,
175 			unsigned int queue, unsigned int off, u64 *val)
176 {
177 	if (!nfp_abm_has_prio(alink->abm)) {
178 		if (!band) {
179 			unsigned int id = alink->queue_base + queue;
180 
181 			*val = nn_readq(alink->vnic,
182 					NFP_NET_CFG_RXR_STATS(id) + off);
183 		} else {
184 			*val = 0;
185 		}
186 
187 		return 0;
188 	} else {
189 		return nfp_abm_ctrl_stat(alink, alink->abm->q_stats,
190 					 NFP_Q_STAT_STRIDE, off, band, queue,
191 					 true, val);
192 	}
193 }
194 
195 int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band,
196 			      unsigned int queue, struct nfp_alink_stats *stats)
197 {
198 	int err;
199 
200 	err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS,
201 				      &stats->tx_pkts);
202 	if (err)
203 		return err;
204 
205 	err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES,
206 				      &stats->tx_bytes);
207 	if (err)
208 		return err;
209 
210 	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE,
211 				NFP_QLVL_BLOG_BYTES, band, queue, false,
212 				&stats->backlog_bytes);
213 	if (err)
214 		return err;
215 
216 	err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls,
217 				NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS,
218 				band, queue, false, &stats->backlog_pkts);
219 	if (err)
220 		return err;
221 
222 	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
223 				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
224 				band, queue, true, &stats->drops);
225 	if (err)
226 		return err;
227 
228 	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
229 				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
230 				 band, queue, true, &stats->overlimits);
231 }
232 
233 int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink,
234 			       unsigned int band, unsigned int queue,
235 			       struct nfp_alink_xstats *xstats)
236 {
237 	int err;
238 
239 	err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
240 				NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP,
241 				band, queue, true, &xstats->pdrop);
242 	if (err)
243 		return err;
244 
245 	return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats,
246 				 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN,
247 				 band, queue, true, &xstats->ecn_marked);
248 }
249 
250 int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm)
251 {
252 	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE,
253 			    NULL, 0, NULL, 0);
254 }
255 
256 int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm)
257 {
258 	return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE,
259 			    NULL, 0, NULL, 0);
260 }
261 
262 int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed)
263 {
264 	const u32 cmd = NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET;
265 	struct nfp_net *nn = alink->vnic;
266 	unsigned int i;
267 	int err;
268 
269 	err = nfp_net_mbox_lock(nn, alink->abm->prio_map_len);
270 	if (err)
271 		return err;
272 
273 	/* Write data_len and wipe reserved */
274 	nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN,
275 		  alink->abm->prio_map_len);
276 
277 	for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32))
278 		nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i,
279 			  packed[i / sizeof(u32)]);
280 
281 	err = nfp_net_mbox_reconfig_and_unlock(nn, cmd);
282 	if (err)
283 		nfp_err(alink->abm->app->cpp,
284 			"setting DSCP -> VQ map failed with error %d\n", err);
285 	return err;
286 }
287 
288 static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink)
289 {
290 	struct nfp_abm *abm = alink->abm;
291 	struct nfp_net *nn = alink->vnic;
292 	unsigned int min_mbox_sz;
293 
294 	if (!nfp_abm_has_prio(alink->abm))
295 		return 0;
296 
297 	min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len;
298 	if (nn->tlv_caps.mbox_len < min_mbox_sz) {
299 		nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n",
300 			nn->tlv_caps.mbox_len,  min_mbox_sz);
301 		return -EINVAL;
302 	}
303 
304 	return 0;
305 }
306 
307 int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink)
308 {
309 	alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ);
310 	alink->queue_base /= alink->vnic->stride_rx;
311 
312 	return nfp_abm_ctrl_prio_check_params(alink);
313 }
314 
315 static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm)
316 {
317 	unsigned int size;
318 
319 	size = roundup_pow_of_two(order_base_2(abm->num_bands));
320 	size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE);
321 	size = round_up(size, sizeof(u32));
322 
323 	return size;
324 }
325 
326 static const struct nfp_rtsym *
327 nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size)
328 {
329 	const struct nfp_rtsym *sym;
330 
331 	sym = nfp_rtsym_lookup(pf->rtbl, name);
332 	if (!sym) {
333 		nfp_err(pf->cpp, "Symbol '%s' not found\n", name);
334 		return ERR_PTR(-ENOENT);
335 	}
336 	if (nfp_rtsym_size(sym) != size) {
337 		nfp_err(pf->cpp,
338 			"Symbol '%s' wrong size: expected %u got %llu\n",
339 			name, size, nfp_rtsym_size(sym));
340 		return ERR_PTR(-EINVAL);
341 	}
342 
343 	return sym;
344 }
345 
346 static const struct nfp_rtsym *
347 nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt,
348 			  size_t size)
349 {
350 	char pf_symbol[64];
351 
352 	size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS);
353 	snprintf(pf_symbol, sizeof(pf_symbol), name_fmt,
354 		 abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : "");
355 
356 	return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size);
357 }
358 
359 int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm)
360 {
361 	struct nfp_pf *pf = abm->app->pf;
362 	const struct nfp_rtsym *sym;
363 	int res;
364 
365 	abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp);
366 
367 	/* Check if Qdisc offloads are supported */
368 	res = nfp_pf_rtsym_read_optional(pf, NFP_RED_SUPPORT_SYM_NAME, 1);
369 	if (res < 0)
370 		return res;
371 	abm->red_support = res;
372 
373 	/* Read count of prios and prio bands */
374 	res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1);
375 	if (res < 0)
376 		return res;
377 	abm->num_bands = res;
378 
379 	res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1);
380 	if (res < 0)
381 		return res;
382 	abm->num_prios = res;
383 
384 	/* Read available actions */
385 	res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME,
386 					 BIT(NFP_ABM_ACT_MARK_DROP));
387 	if (res < 0)
388 		return res;
389 	abm->action_mask = res;
390 
391 	abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm);
392 	abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios));
393 
394 	/* Check values are sane, U16_MAX is arbitrarily chosen as max */
395 	if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) ||
396 	    abm->num_bands > U16_MAX || abm->num_prios > U16_MAX ||
397 	    (abm->num_bands == 1) != (abm->num_prios == 1)) {
398 		nfp_err(pf->cpp,
399 			"invalid priomap description num bands: %u and num prios: %u\n",
400 			abm->num_bands, abm->num_prios);
401 		return -EINVAL;
402 	}
403 
404 	/* Find level and stat symbols */
405 	if (!abm->red_support)
406 		return 0;
407 
408 	sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME,
409 					NFP_QLVL_STRIDE);
410 	if (IS_ERR(sym))
411 		return PTR_ERR(sym);
412 	abm->q_lvls = sym;
413 
414 	sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME,
415 					NFP_QMSTAT_STRIDE);
416 	if (IS_ERR(sym))
417 		return PTR_ERR(sym);
418 	abm->qm_stats = sym;
419 
420 	if (nfp_abm_has_prio(abm)) {
421 		sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME,
422 						NFP_Q_STAT_STRIDE);
423 		if (IS_ERR(sym))
424 			return PTR_ERR(sym);
425 		abm->q_stats = sym;
426 	}
427 
428 	return 0;
429 }
430