1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Atlantic Network Driver
3  *
4  * Copyright (C) 2014-2019 aQuantia Corporation
5  * Copyright (C) 2019-2020 Marvell International Ltd.
6  */
7 
8 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings.
9  * Definition of functions for Rx and Tx rings. Friendly module for aq_nic.
10  */
11 
12 #include "aq_vec.h"
13 #include "aq_nic.h"
14 #include "aq_ring.h"
15 #include "aq_hw.h"
16 
17 #include <linux/netdevice.h>
18 
19 struct aq_vec_s {
20 	const struct aq_hw_ops *aq_hw_ops;
21 	struct aq_hw_s *aq_hw;
22 	struct aq_nic_s *aq_nic;
23 	unsigned int tx_rings;
24 	unsigned int rx_rings;
25 	struct aq_ring_param_s aq_ring_param;
26 	struct napi_struct napi;
27 	struct aq_ring_s ring[AQ_CFG_TCS_MAX][2];
28 };
29 
30 #define AQ_VEC_TX_ID 0
31 #define AQ_VEC_RX_ID 1
32 
33 static int aq_vec_poll(struct napi_struct *napi, int budget)
34 {
35 	struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
36 	unsigned int sw_tail_old = 0U;
37 	struct aq_ring_s *ring = NULL;
38 	bool was_tx_cleaned = true;
39 	unsigned int i = 0U;
40 	int work_done = 0;
41 	int err = 0;
42 
43 	if (!self) {
44 		err = -EINVAL;
45 	} else {
46 		for (i = 0U, ring = self->ring[0];
47 			self->tx_rings > i; ++i, ring = self->ring[i]) {
48 			u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
49 			ring[AQ_VEC_RX_ID].stats.rx.polls++;
50 			u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
51 			if (self->aq_hw_ops->hw_ring_tx_head_update) {
52 				err = self->aq_hw_ops->hw_ring_tx_head_update(
53 							self->aq_hw,
54 							&ring[AQ_VEC_TX_ID]);
55 				if (err < 0)
56 					goto err_exit;
57 			}
58 
59 			if (ring[AQ_VEC_TX_ID].sw_head !=
60 			    ring[AQ_VEC_TX_ID].hw_head) {
61 				was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
62 				aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
63 			}
64 
65 			err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
66 					    &ring[AQ_VEC_RX_ID]);
67 			if (err < 0)
68 				goto err_exit;
69 
70 			if (ring[AQ_VEC_RX_ID].sw_head !=
71 				ring[AQ_VEC_RX_ID].hw_head) {
72 				err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
73 						       napi,
74 						       &work_done,
75 						       budget - work_done);
76 				if (err < 0)
77 					goto err_exit;
78 
79 				sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail;
80 
81 				err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
82 				if (err < 0)
83 					goto err_exit;
84 
85 				err = self->aq_hw_ops->hw_ring_rx_fill(
86 					self->aq_hw,
87 					&ring[AQ_VEC_RX_ID], sw_tail_old);
88 				if (err < 0)
89 					goto err_exit;
90 			}
91 		}
92 
93 err_exit:
94 		if (!was_tx_cleaned)
95 			work_done = budget;
96 
97 		if (work_done < budget) {
98 			napi_complete_done(napi, work_done);
99 			self->aq_hw_ops->hw_irq_enable(self->aq_hw,
100 					1U << self->aq_ring_param.vec_idx);
101 		}
102 	}
103 
104 	return work_done;
105 }
106 
107 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
108 			      struct aq_nic_cfg_s *aq_nic_cfg)
109 {
110 	struct aq_vec_s *self = NULL;
111 
112 	self = kzalloc(sizeof(*self), GFP_KERNEL);
113 	if (!self)
114 		goto err_exit;
115 
116 	self->aq_nic = aq_nic;
117 	self->aq_ring_param.vec_idx = idx;
118 	self->aq_ring_param.cpu =
119 		idx + aq_nic_cfg->aq_rss.base_cpu_number;
120 
121 	cpumask_set_cpu(self->aq_ring_param.cpu,
122 			&self->aq_ring_param.affinity_mask);
123 
124 	self->tx_rings = 0;
125 	self->rx_rings = 0;
126 
127 	netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi,
128 		       aq_vec_poll, AQ_CFG_NAPI_WEIGHT);
129 
130 err_exit:
131 	return self;
132 }
133 
134 int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic,
135 		      unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg)
136 {
137 	struct aq_ring_s *ring = NULL;
138 	unsigned int i = 0U;
139 	int err = 0;
140 
141 	for (i = 0; i < aq_nic_cfg->tcs; ++i) {
142 		const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg,
143 								    i, idx);
144 
145 		ring = aq_ring_tx_alloc(&self->ring[i][AQ_VEC_TX_ID], aq_nic,
146 					idx_ring, aq_nic_cfg);
147 		if (!ring) {
148 			err = -ENOMEM;
149 			goto err_exit;
150 		}
151 
152 		++self->tx_rings;
153 
154 		aq_nic_set_tx_ring(aq_nic, idx_ring, ring);
155 
156 		ring = aq_ring_rx_alloc(&self->ring[i][AQ_VEC_RX_ID], aq_nic,
157 					idx_ring, aq_nic_cfg);
158 		if (!ring) {
159 			err = -ENOMEM;
160 			goto err_exit;
161 		}
162 
163 		++self->rx_rings;
164 	}
165 
166 err_exit:
167 	if (err < 0) {
168 		aq_vec_ring_free(self);
169 		self = NULL;
170 	}
171 
172 	return err;
173 }
174 
175 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
176 		struct aq_hw_s *aq_hw)
177 {
178 	struct aq_ring_s *ring = NULL;
179 	unsigned int i = 0U;
180 	int err = 0;
181 
182 	self->aq_hw_ops = aq_hw_ops;
183 	self->aq_hw = aq_hw;
184 
185 	for (i = 0U, ring = self->ring[0];
186 		self->tx_rings > i; ++i, ring = self->ring[i]) {
187 		err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX);
188 		if (err < 0)
189 			goto err_exit;
190 
191 		err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw,
192 						       &ring[AQ_VEC_TX_ID],
193 						       &self->aq_ring_param);
194 		if (err < 0)
195 			goto err_exit;
196 
197 		err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX);
198 		if (err < 0)
199 			goto err_exit;
200 
201 		err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw,
202 						       &ring[AQ_VEC_RX_ID],
203 						       &self->aq_ring_param);
204 		if (err < 0)
205 			goto err_exit;
206 
207 		err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
208 		if (err < 0)
209 			goto err_exit;
210 
211 		err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw,
212 						       &ring[AQ_VEC_RX_ID], 0U);
213 		if (err < 0)
214 			goto err_exit;
215 	}
216 
217 err_exit:
218 	return err;
219 }
220 
221 int aq_vec_start(struct aq_vec_s *self)
222 {
223 	struct aq_ring_s *ring = NULL;
224 	unsigned int i = 0U;
225 	int err = 0;
226 
227 	for (i = 0U, ring = self->ring[0];
228 		self->tx_rings > i; ++i, ring = self->ring[i]) {
229 		err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
230 							&ring[AQ_VEC_TX_ID]);
231 		if (err < 0)
232 			goto err_exit;
233 
234 		err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw,
235 							&ring[AQ_VEC_RX_ID]);
236 		if (err < 0)
237 			goto err_exit;
238 	}
239 
240 	napi_enable(&self->napi);
241 
242 err_exit:
243 	return err;
244 }
245 
246 void aq_vec_stop(struct aq_vec_s *self)
247 {
248 	struct aq_ring_s *ring = NULL;
249 	unsigned int i = 0U;
250 
251 	for (i = 0U, ring = self->ring[0];
252 		self->tx_rings > i; ++i, ring = self->ring[i]) {
253 		self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
254 						 &ring[AQ_VEC_TX_ID]);
255 
256 		self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw,
257 						 &ring[AQ_VEC_RX_ID]);
258 	}
259 
260 	napi_disable(&self->napi);
261 }
262 
263 void aq_vec_deinit(struct aq_vec_s *self)
264 {
265 	struct aq_ring_s *ring = NULL;
266 	unsigned int i = 0U;
267 
268 	if (!self)
269 		goto err_exit;
270 
271 	for (i = 0U, ring = self->ring[0];
272 		self->tx_rings > i; ++i, ring = self->ring[i]) {
273 		aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
274 		aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
275 	}
276 
277 err_exit:;
278 }
279 
280 void aq_vec_free(struct aq_vec_s *self)
281 {
282 	if (!self)
283 		goto err_exit;
284 
285 	netif_napi_del(&self->napi);
286 
287 	kfree(self);
288 
289 err_exit:;
290 }
291 
292 void aq_vec_ring_free(struct aq_vec_s *self)
293 {
294 	struct aq_ring_s *ring = NULL;
295 	unsigned int i = 0U;
296 
297 	if (!self)
298 		goto err_exit;
299 
300 	for (i = 0U, ring = self->ring[0];
301 		self->tx_rings > i; ++i, ring = self->ring[i]) {
302 		aq_ring_free(&ring[AQ_VEC_TX_ID]);
303 		if (i < self->rx_rings)
304 			aq_ring_free(&ring[AQ_VEC_RX_ID]);
305 	}
306 
307 	self->tx_rings = 0;
308 	self->rx_rings = 0;
309 err_exit:;
310 }
311 
312 irqreturn_t aq_vec_isr(int irq, void *private)
313 {
314 	struct aq_vec_s *self = private;
315 	int err = 0;
316 
317 	if (!self) {
318 		err = -EINVAL;
319 		goto err_exit;
320 	}
321 	napi_schedule(&self->napi);
322 
323 err_exit:
324 	return err >= 0 ? IRQ_HANDLED : IRQ_NONE;
325 }
326 
327 irqreturn_t aq_vec_isr_legacy(int irq, void *private)
328 {
329 	struct aq_vec_s *self = private;
330 	u64 irq_mask = 0U;
331 	int err;
332 
333 	if (!self)
334 		return IRQ_NONE;
335 	err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask);
336 	if (err < 0)
337 		return IRQ_NONE;
338 
339 	if (irq_mask) {
340 		self->aq_hw_ops->hw_irq_disable(self->aq_hw,
341 			      1U << self->aq_ring_param.vec_idx);
342 		napi_schedule(&self->napi);
343 	} else {
344 		self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U);
345 		return IRQ_NONE;
346 	}
347 
348 	return IRQ_HANDLED;
349 }
350 
351 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self)
352 {
353 	return &self->aq_ring_param.affinity_mask;
354 }
355 
356 bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc)
357 {
358 	return tc < self->rx_rings && tc < self->tx_rings;
359 }
360 
361 unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data)
362 {
363 	unsigned int count;
364 
365 	if (!aq_vec_is_valid_tc(self, tc))
366 		return 0;
367 
368 	count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data);
369 	count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count);
370 
371 	return count;
372 }
373