1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Atlantic Network Driver
3  *
4  * Copyright (C) 2014-2019 aQuantia Corporation
5  * Copyright (C) 2019-2020 Marvell International Ltd.
6  */
7 
8 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings.
9  * Definition of functions for Rx and Tx rings. Friendly module for aq_nic.
10  */
11 
12 #include "aq_vec.h"
13 
14 struct aq_vec_s {
15 	const struct aq_hw_ops *aq_hw_ops;
16 	struct aq_hw_s *aq_hw;
17 	struct aq_nic_s *aq_nic;
18 	unsigned int tx_rings;
19 	unsigned int rx_rings;
20 	struct aq_ring_param_s aq_ring_param;
21 	struct napi_struct napi;
22 	struct aq_ring_s ring[AQ_CFG_TCS_MAX][2];
23 };
24 
25 #define AQ_VEC_TX_ID 0
26 #define AQ_VEC_RX_ID 1
27 
28 static int aq_vec_poll(struct napi_struct *napi, int budget)
29 {
30 	struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
31 	unsigned int sw_tail_old = 0U;
32 	struct aq_ring_s *ring = NULL;
33 	bool was_tx_cleaned = true;
34 	unsigned int i = 0U;
35 	int work_done = 0;
36 	int err = 0;
37 
38 	if (!self) {
39 		err = -EINVAL;
40 	} else {
41 		for (i = 0U; self->tx_rings > i; ++i) {
42 			ring = self->ring[i];
43 			u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
44 			ring[AQ_VEC_RX_ID].stats.rx.polls++;
45 			u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
46 			if (self->aq_hw_ops->hw_ring_tx_head_update) {
47 				err = self->aq_hw_ops->hw_ring_tx_head_update(
48 							self->aq_hw,
49 							&ring[AQ_VEC_TX_ID]);
50 				if (err < 0)
51 					goto err_exit;
52 			}
53 
54 			if (ring[AQ_VEC_TX_ID].sw_head !=
55 			    ring[AQ_VEC_TX_ID].hw_head) {
56 				was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
57 				aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
58 			}
59 
60 			err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
61 					    &ring[AQ_VEC_RX_ID]);
62 			if (err < 0)
63 				goto err_exit;
64 
65 			if (ring[AQ_VEC_RX_ID].sw_head !=
66 				ring[AQ_VEC_RX_ID].hw_head) {
67 				err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
68 						       napi,
69 						       &work_done,
70 						       budget - work_done);
71 				if (err < 0)
72 					goto err_exit;
73 
74 				sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail;
75 
76 				err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
77 				if (err < 0)
78 					goto err_exit;
79 
80 				err = self->aq_hw_ops->hw_ring_rx_fill(
81 					self->aq_hw,
82 					&ring[AQ_VEC_RX_ID], sw_tail_old);
83 				if (err < 0)
84 					goto err_exit;
85 			}
86 		}
87 
88 err_exit:
89 		if (!was_tx_cleaned)
90 			work_done = budget;
91 
92 		if (work_done < budget) {
93 			napi_complete_done(napi, work_done);
94 			self->aq_hw_ops->hw_irq_enable(self->aq_hw,
95 					1U << self->aq_ring_param.vec_idx);
96 		}
97 	}
98 
99 	return work_done;
100 }
101 
102 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
103 			      struct aq_nic_cfg_s *aq_nic_cfg)
104 {
105 	struct aq_vec_s *self = NULL;
106 
107 	self = kzalloc(sizeof(*self), GFP_KERNEL);
108 	if (!self)
109 		goto err_exit;
110 
111 	self->aq_nic = aq_nic;
112 	self->aq_ring_param.vec_idx = idx;
113 	self->aq_ring_param.cpu =
114 		idx + aq_nic_cfg->aq_rss.base_cpu_number;
115 
116 	cpumask_set_cpu(self->aq_ring_param.cpu,
117 			&self->aq_ring_param.affinity_mask);
118 
119 	self->tx_rings = 0;
120 	self->rx_rings = 0;
121 
122 	netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll);
123 
124 err_exit:
125 	return self;
126 }
127 
128 int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic,
129 		      unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg)
130 {
131 	struct aq_ring_s *ring = NULL;
132 	unsigned int i = 0U;
133 	int err = 0;
134 
135 	for (i = 0; i < aq_nic_cfg->tcs; ++i) {
136 		const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg,
137 								    i, idx);
138 
139 		ring = &self->ring[i][AQ_VEC_TX_ID];
140 		err = aq_ring_tx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
141 		if (err)
142 			goto err_exit;
143 
144 		++self->tx_rings;
145 
146 		aq_nic_set_tx_ring(aq_nic, idx_ring, ring);
147 
148 		ring = &self->ring[i][AQ_VEC_RX_ID];
149 		if (xdp_rxq_info_reg(&ring->xdp_rxq,
150 				     aq_nic->ndev, idx,
151 				     self->napi.napi_id) < 0) {
152 			err = -ENOMEM;
153 			goto err_exit;
154 		}
155 		if (xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
156 					       MEM_TYPE_PAGE_SHARED, NULL) < 0) {
157 			xdp_rxq_info_unreg(&ring->xdp_rxq);
158 			err = -ENOMEM;
159 			goto err_exit;
160 		}
161 
162 		err = aq_ring_rx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
163 		if (err) {
164 			xdp_rxq_info_unreg(&ring->xdp_rxq);
165 			goto err_exit;
166 		}
167 
168 		++self->rx_rings;
169 	}
170 
171 err_exit:
172 	if (err < 0) {
173 		aq_vec_ring_free(self);
174 		self = NULL;
175 	}
176 
177 	return err;
178 }
179 
180 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
181 		struct aq_hw_s *aq_hw)
182 {
183 	struct aq_ring_s *ring = NULL;
184 	unsigned int i = 0U;
185 	int err = 0;
186 
187 	self->aq_hw_ops = aq_hw_ops;
188 	self->aq_hw = aq_hw;
189 
190 	for (i = 0U; self->tx_rings > i; ++i) {
191 		ring = self->ring[i];
192 		err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX);
193 		if (err < 0)
194 			goto err_exit;
195 
196 		err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw,
197 						       &ring[AQ_VEC_TX_ID],
198 						       &self->aq_ring_param);
199 		if (err < 0)
200 			goto err_exit;
201 
202 		err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX);
203 		if (err < 0)
204 			goto err_exit;
205 
206 		err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw,
207 						       &ring[AQ_VEC_RX_ID],
208 						       &self->aq_ring_param);
209 		if (err < 0)
210 			goto err_exit;
211 
212 		err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
213 		if (err < 0)
214 			goto err_exit;
215 
216 		err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw,
217 						       &ring[AQ_VEC_RX_ID], 0U);
218 		if (err < 0)
219 			goto err_exit;
220 	}
221 
222 err_exit:
223 	return err;
224 }
225 
226 int aq_vec_start(struct aq_vec_s *self)
227 {
228 	struct aq_ring_s *ring = NULL;
229 	unsigned int i = 0U;
230 	int err = 0;
231 
232 	for (i = 0U; self->tx_rings > i; ++i) {
233 		ring = self->ring[i];
234 		err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
235 							&ring[AQ_VEC_TX_ID]);
236 		if (err < 0)
237 			goto err_exit;
238 
239 		err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw,
240 							&ring[AQ_VEC_RX_ID]);
241 		if (err < 0)
242 			goto err_exit;
243 	}
244 
245 	napi_enable(&self->napi);
246 
247 err_exit:
248 	return err;
249 }
250 
251 void aq_vec_stop(struct aq_vec_s *self)
252 {
253 	struct aq_ring_s *ring = NULL;
254 	unsigned int i = 0U;
255 
256 	for (i = 0U; self->tx_rings > i; ++i) {
257 		ring = self->ring[i];
258 		self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
259 						 &ring[AQ_VEC_TX_ID]);
260 
261 		self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw,
262 						 &ring[AQ_VEC_RX_ID]);
263 	}
264 
265 	napi_disable(&self->napi);
266 }
267 
268 void aq_vec_deinit(struct aq_vec_s *self)
269 {
270 	struct aq_ring_s *ring = NULL;
271 	unsigned int i = 0U;
272 
273 	if (!self)
274 		goto err_exit;
275 
276 	for (i = 0U; self->tx_rings > i; ++i) {
277 		ring = self->ring[i];
278 		aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
279 		aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
280 	}
281 
282 err_exit:;
283 }
284 
285 void aq_vec_free(struct aq_vec_s *self)
286 {
287 	if (!self)
288 		goto err_exit;
289 
290 	netif_napi_del(&self->napi);
291 
292 	kfree(self);
293 
294 err_exit:;
295 }
296 
297 void aq_vec_ring_free(struct aq_vec_s *self)
298 {
299 	struct aq_ring_s *ring = NULL;
300 	unsigned int i = 0U;
301 
302 	if (!self)
303 		goto err_exit;
304 
305 	for (i = 0U; self->tx_rings > i; ++i) {
306 		ring = self->ring[i];
307 		aq_ring_free(&ring[AQ_VEC_TX_ID]);
308 		if (i < self->rx_rings) {
309 			xdp_rxq_info_unreg(&ring[AQ_VEC_RX_ID].xdp_rxq);
310 			aq_ring_free(&ring[AQ_VEC_RX_ID]);
311 		}
312 	}
313 
314 	self->tx_rings = 0;
315 	self->rx_rings = 0;
316 err_exit:;
317 }
318 
319 irqreturn_t aq_vec_isr(int irq, void *private)
320 {
321 	struct aq_vec_s *self = private;
322 	int err = 0;
323 
324 	if (!self) {
325 		err = -EINVAL;
326 		goto err_exit;
327 	}
328 	napi_schedule(&self->napi);
329 
330 err_exit:
331 	return err >= 0 ? IRQ_HANDLED : IRQ_NONE;
332 }
333 
334 irqreturn_t aq_vec_isr_legacy(int irq, void *private)
335 {
336 	struct aq_vec_s *self = private;
337 	u64 irq_mask = 0U;
338 	int err;
339 
340 	if (!self)
341 		return IRQ_NONE;
342 	err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask);
343 	if (err < 0)
344 		return IRQ_NONE;
345 
346 	if (irq_mask) {
347 		self->aq_hw_ops->hw_irq_disable(self->aq_hw,
348 			      1U << self->aq_ring_param.vec_idx);
349 		napi_schedule(&self->napi);
350 	} else {
351 		self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U);
352 		return IRQ_NONE;
353 	}
354 
355 	return IRQ_HANDLED;
356 }
357 
358 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self)
359 {
360 	return &self->aq_ring_param.affinity_mask;
361 }
362 
363 bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc)
364 {
365 	return tc < self->rx_rings && tc < self->tx_rings;
366 }
367 
368 unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data)
369 {
370 	unsigned int count;
371 
372 	if (!aq_vec_is_valid_tc(self, tc))
373 		return 0;
374 
375 	count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data);
376 	count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count);
377 
378 	return count;
379 }
380