1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2 
3 /* Packet receive logic for Mellanox Gigabit Ethernet driver
4  *
5  * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
6  */
7 
8 #include <linux/etherdevice.h>
9 #include <linux/skbuff.h>
10 
11 #include "mlxbf_gige.h"
12 #include "mlxbf_gige_regs.h"
13 
14 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
15 				  unsigned int index, u64 dmac)
16 {
17 	void __iomem *base = priv->base;
18 	u64 control;
19 
20 	/* Write destination MAC to specified MAC RX filter */
21 	writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
22 	       (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
23 
24 	/* Enable MAC receive filter mask for specified index */
25 	control = readq(base + MLXBF_GIGE_CONTROL);
26 	control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
27 	writeq(control, base + MLXBF_GIGE_CONTROL);
28 }
29 
30 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
31 				  unsigned int index, u64 *dmac)
32 {
33 	void __iomem *base = priv->base;
34 
35 	/* Read destination MAC from specified MAC RX filter */
36 	*dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
37 		      (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
38 }
39 
40 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
41 {
42 	void __iomem *base = priv->base;
43 	u64 control;
44 	u64 end_mac;
45 
46 	/* Enable MAC_ID_RANGE match functionality */
47 	control = readq(base + MLXBF_GIGE_CONTROL);
48 	control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
49 	writeq(control, base + MLXBF_GIGE_CONTROL);
50 
51 	/* Set start of destination MAC range check to 0 */
52 	writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);
53 
54 	/* Set end of destination MAC range check to all FFs */
55 	end_mac = BCAST_MAC_ADDR;
56 	writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
57 }
58 
59 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
60 {
61 	void __iomem *base = priv->base;
62 	u64 control;
63 
64 	/* Disable MAC_ID_RANGE match functionality */
65 	control = readq(base + MLXBF_GIGE_CONTROL);
66 	control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
67 	writeq(control, base + MLXBF_GIGE_CONTROL);
68 
69 	/* NOTE: no need to change DMAC_RANGE_START or END;
70 	 * those values are ignored since MAC_ID_RANGE_EN=0
71 	 */
72 }
73 
74 /* Receive Initialization
75  * 1) Configures RX MAC filters via MMIO registers
76  * 2) Allocates RX WQE array using coherent DMA mapping
77  * 3) Initializes each element of RX WQE array with a receive
78  *    buffer pointer (also using coherent DMA mapping)
79  * 4) Allocates RX CQE array using coherent DMA mapping
80  * 5) Completes other misc receive initialization
81  */
82 int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
83 {
84 	size_t wq_size, cq_size;
85 	dma_addr_t *rx_wqe_ptr;
86 	dma_addr_t rx_buf_dma;
87 	u64 data;
88 	int i, j;
89 
90 	/* Configure MAC RX filter #0 to allow RX of broadcast pkts */
91 	mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
92 				     BCAST_MAC_ADDR);
93 
94 	wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
95 	priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
96 					       &priv->rx_wqe_base_dma,
97 					       GFP_KERNEL);
98 	if (!priv->rx_wqe_base)
99 		return -ENOMEM;
100 
101 	/* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
102 	 * Each RX WQE is simply a receive buffer pointer, so walk
103 	 * the entire array, allocating a 2KB buffer for each element
104 	 */
105 	rx_wqe_ptr = priv->rx_wqe_base;
106 
107 	for (i = 0; i < priv->rx_q_entries; i++) {
108 		priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
109 						       &rx_buf_dma, DMA_FROM_DEVICE);
110 		if (!priv->rx_skb[i])
111 			goto free_wqe_and_skb;
112 		*rx_wqe_ptr++ = rx_buf_dma;
113 	}
114 
115 	/* Write RX WQE base address into MMIO reg */
116 	writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);
117 
118 	cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
119 	priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
120 					       &priv->rx_cqe_base_dma,
121 					       GFP_KERNEL);
122 	if (!priv->rx_cqe_base)
123 		goto free_wqe_and_skb;
124 
125 	for (i = 0; i < priv->rx_q_entries; i++)
126 		priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;
127 
128 	/* Write RX CQE base address into MMIO reg */
129 	writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);
130 
131 	/* Write RX_WQE_PI with current number of replenished buffers */
132 	writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);
133 
134 	/* Enable removal of CRC during RX */
135 	data = readq(priv->base + MLXBF_GIGE_RX);
136 	data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
137 	writeq(data, priv->base + MLXBF_GIGE_RX);
138 
139 	/* Enable RX MAC filter pass and discard counters */
140 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
141 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
142 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
143 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);
144 
145 	writeq(ilog2(priv->rx_q_entries),
146 	       priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);
147 
148 	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
149 	 * indicate readiness to receive interrupts
150 	 */
151 	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
152 	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
153 	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
154 
155 	/* Enable RX DMA to write new packets to memory */
156 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
157 	data |= MLXBF_GIGE_RX_DMA_EN;
158 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
159 
160 	return 0;
161 
162 free_wqe_and_skb:
163 	rx_wqe_ptr = priv->rx_wqe_base;
164 	for (j = 0; j < i; j++) {
165 		dma_unmap_single(priv->dev, *rx_wqe_ptr,
166 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
167 		dev_kfree_skb(priv->rx_skb[j]);
168 		rx_wqe_ptr++;
169 	}
170 	dma_free_coherent(priv->dev, wq_size,
171 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
172 	return -ENOMEM;
173 }
174 
175 /* Receive Deinitialization
176  * This routine will free allocations done by mlxbf_gige_rx_init(),
177  * namely the RX WQE and RX CQE arrays, as well as all RX buffers
178  */
179 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
180 {
181 	dma_addr_t *rx_wqe_ptr;
182 	size_t size;
183 	u64 data;
184 	int i;
185 
186 	/* Disable RX DMA to prevent packet transfers to memory */
187 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
188 	data &= ~MLXBF_GIGE_RX_DMA_EN;
189 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
190 
191 	rx_wqe_ptr = priv->rx_wqe_base;
192 
193 	for (i = 0; i < priv->rx_q_entries; i++) {
194 		dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
195 				 DMA_FROM_DEVICE);
196 		dev_kfree_skb(priv->rx_skb[i]);
197 		rx_wqe_ptr++;
198 	}
199 
200 	size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
201 	dma_free_coherent(priv->dev, size,
202 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
203 
204 	size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
205 	dma_free_coherent(priv->dev, size,
206 			  priv->rx_cqe_base, priv->rx_cqe_base_dma);
207 
208 	priv->rx_wqe_base = NULL;
209 	priv->rx_wqe_base_dma = 0;
210 	priv->rx_cqe_base = NULL;
211 	priv->rx_cqe_base_dma = 0;
212 	writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
213 	writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
214 }
215 
216 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
217 {
218 	struct net_device *netdev = priv->netdev;
219 	struct sk_buff *skb = NULL, *rx_skb;
220 	u16 rx_pi_rem, rx_ci_rem;
221 	dma_addr_t *rx_wqe_addr;
222 	dma_addr_t rx_buf_dma;
223 	u64 *rx_cqe_addr;
224 	u64 datalen;
225 	u64 rx_cqe;
226 	u16 rx_ci;
227 	u16 rx_pi;
228 
229 	/* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
230 	rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
231 	rx_pi_rem = rx_pi % priv->rx_q_entries;
232 
233 	rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
234 	rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
235 	rx_cqe = *rx_cqe_addr;
236 
237 	if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
238 		return false;
239 
240 	if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
241 		/* Packet is OK, increment stats */
242 		datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
243 		netdev->stats.rx_packets++;
244 		netdev->stats.rx_bytes += datalen;
245 
246 		skb = priv->rx_skb[rx_pi_rem];
247 
248 		/* Alloc another RX SKB for this same index */
249 		rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
250 					      &rx_buf_dma, DMA_FROM_DEVICE);
251 		if (!rx_skb)
252 			return false;
253 		priv->rx_skb[rx_pi_rem] = rx_skb;
254 		dma_unmap_single(priv->dev, *rx_wqe_addr,
255 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
256 
257 		skb_put(skb, datalen);
258 
259 		skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */
260 
261 		skb->protocol = eth_type_trans(skb, netdev);
262 
263 		*rx_wqe_addr = rx_buf_dma;
264 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
265 		priv->stats.rx_mac_errors++;
266 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
267 		priv->stats.rx_truncate_errors++;
268 	}
269 
270 	/* Read receive consumer index before replenish so that this routine
271 	 * returns accurate return value even if packet is received into
272 	 * just-replenished buffer prior to exiting this routine.
273 	 */
274 	rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
275 	rx_ci_rem = rx_ci % priv->rx_q_entries;
276 
277 	/* Let hardware know we've replenished one buffer */
278 	rx_pi++;
279 
280 	/* Ensure completion of all writes before notifying HW of replenish */
281 	wmb();
282 	writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);
283 
284 	(*rx_pkts)++;
285 
286 	rx_pi_rem = rx_pi % priv->rx_q_entries;
287 	if (rx_pi_rem == 0)
288 		priv->valid_polarity ^= 1;
289 
290 	if (skb)
291 		netif_receive_skb(skb);
292 
293 	return rx_pi_rem != rx_ci_rem;
294 }
295 
296 /* Driver poll() function called by NAPI infrastructure */
297 int mlxbf_gige_poll(struct napi_struct *napi, int budget)
298 {
299 	struct mlxbf_gige *priv;
300 	bool remaining_pkts;
301 	int work_done = 0;
302 	u64 data;
303 
304 	priv = container_of(napi, struct mlxbf_gige, napi);
305 
306 	mlxbf_gige_handle_tx_complete(priv);
307 
308 	do {
309 		remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
310 	} while (remaining_pkts && work_done < budget);
311 
312 	/* If amount of work done < budget, turn off NAPI polling
313 	 * via napi_complete_done(napi, work_done) and then
314 	 * re-enable interrupts.
315 	 */
316 	if (work_done < budget && napi_complete_done(napi, work_done)) {
317 		/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
318 		 * indicate receive readiness
319 		 */
320 		data = readq(priv->base + MLXBF_GIGE_INT_MASK);
321 		data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
322 		writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
323 	}
324 
325 	return work_done;
326 }
327