1 /*
2  * Linux driver for VMware's vmxnet3 ethernet NIC.
3  *
4  * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation; version 2 of the License and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13  * NON INFRINGEMENT.  See the GNU General Public License for more
14  * details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19  *
20  * The full GNU General Public License is included in this distribution in
21  * the file called "COPYING".
22  *
23  * Maintained by: pv-drivers@vmware.com
24  *
25  */
26 
27 
28 #include "vmxnet3_int.h"
29 #include <net/vxlan.h>
30 #include <net/geneve.h>
31 #include "vmxnet3_xdp.h"
32 
33 #define VXLAN_UDP_PORT 8472
34 
35 struct vmxnet3_stat_desc {
36 	char desc[ETH_GSTRING_LEN];
37 	int  offset;
38 };
39 
40 
41 /* per tq stats maintained by the device */
42 static const struct vmxnet3_stat_desc
43 vmxnet3_tq_dev_stats[] = {
44 	/* description,         offset */
45 	{ "Tx Queue#",        0 },
46 	{ "  TSO pkts tx",	offsetof(struct UPT1_TxStats, TSOPktsTxOK) },
47 	{ "  TSO bytes tx",	offsetof(struct UPT1_TxStats, TSOBytesTxOK) },
48 	{ "  ucast pkts tx",	offsetof(struct UPT1_TxStats, ucastPktsTxOK) },
49 	{ "  ucast bytes tx",	offsetof(struct UPT1_TxStats, ucastBytesTxOK) },
50 	{ "  mcast pkts tx",	offsetof(struct UPT1_TxStats, mcastPktsTxOK) },
51 	{ "  mcast bytes tx",	offsetof(struct UPT1_TxStats, mcastBytesTxOK) },
52 	{ "  bcast pkts tx",	offsetof(struct UPT1_TxStats, bcastPktsTxOK) },
53 	{ "  bcast bytes tx",	offsetof(struct UPT1_TxStats, bcastBytesTxOK) },
54 	{ "  pkts tx err",	offsetof(struct UPT1_TxStats, pktsTxError) },
55 	{ "  pkts tx discard",	offsetof(struct UPT1_TxStats, pktsTxDiscard) },
56 };
57 
58 /* per tq stats maintained by the driver */
59 static const struct vmxnet3_stat_desc
60 vmxnet3_tq_driver_stats[] = {
61 	/* description,         offset */
62 	{"  drv dropped tx total",	offsetof(struct vmxnet3_tq_driver_stats,
63 						 drop_total) },
64 	{ "     too many frags", offsetof(struct vmxnet3_tq_driver_stats,
65 					  drop_too_many_frags) },
66 	{ "     giant hdr",	offsetof(struct vmxnet3_tq_driver_stats,
67 					 drop_oversized_hdr) },
68 	{ "     hdr err",	offsetof(struct vmxnet3_tq_driver_stats,
69 					 drop_hdr_inspect_err) },
70 	{ "     tso",		offsetof(struct vmxnet3_tq_driver_stats,
71 					 drop_tso) },
72 	{ "  ring full",	offsetof(struct vmxnet3_tq_driver_stats,
73 					 tx_ring_full) },
74 	{ "  pkts linearized",	offsetof(struct vmxnet3_tq_driver_stats,
75 					 linearized) },
76 	{ "  hdr cloned",	offsetof(struct vmxnet3_tq_driver_stats,
77 					 copy_skb_header) },
78 	{ "  giant hdr",	offsetof(struct vmxnet3_tq_driver_stats,
79 					 oversized_hdr) },
80 	{ "  xdp xmit",		offsetof(struct vmxnet3_tq_driver_stats,
81 					 xdp_xmit) },
82 	{ "  xdp xmit err",	offsetof(struct vmxnet3_tq_driver_stats,
83 					 xdp_xmit_err) },
84 };
85 
86 /* per rq stats maintained by the device */
87 static const struct vmxnet3_stat_desc
88 vmxnet3_rq_dev_stats[] = {
89 	{ "Rx Queue#",        0 },
90 	{ "  LRO pkts rx",	offsetof(struct UPT1_RxStats, LROPktsRxOK) },
91 	{ "  LRO byte rx",	offsetof(struct UPT1_RxStats, LROBytesRxOK) },
92 	{ "  ucast pkts rx",	offsetof(struct UPT1_RxStats, ucastPktsRxOK) },
93 	{ "  ucast bytes rx",	offsetof(struct UPT1_RxStats, ucastBytesRxOK) },
94 	{ "  mcast pkts rx",	offsetof(struct UPT1_RxStats, mcastPktsRxOK) },
95 	{ "  mcast bytes rx",	offsetof(struct UPT1_RxStats, mcastBytesRxOK) },
96 	{ "  bcast pkts rx",	offsetof(struct UPT1_RxStats, bcastPktsRxOK) },
97 	{ "  bcast bytes rx",	offsetof(struct UPT1_RxStats, bcastBytesRxOK) },
98 	{ "  pkts rx OOB",	offsetof(struct UPT1_RxStats, pktsRxOutOfBuf) },
99 	{ "  pkts rx err",	offsetof(struct UPT1_RxStats, pktsRxError) },
100 };
101 
102 /* per rq stats maintained by the driver */
103 static const struct vmxnet3_stat_desc
104 vmxnet3_rq_driver_stats[] = {
105 	/* description,         offset */
106 	{ "  drv dropped rx total", offsetof(struct vmxnet3_rq_driver_stats,
107 					     drop_total) },
108 	{ "     err",		offsetof(struct vmxnet3_rq_driver_stats,
109 					 drop_err) },
110 	{ "     fcs",		offsetof(struct vmxnet3_rq_driver_stats,
111 					 drop_fcs) },
112 	{ "  rx buf alloc fail", offsetof(struct vmxnet3_rq_driver_stats,
113 					  rx_buf_alloc_failure) },
114 	{ "     xdp packets", offsetof(struct vmxnet3_rq_driver_stats,
115 				       xdp_packets) },
116 	{ "     xdp tx", offsetof(struct vmxnet3_rq_driver_stats,
117 				  xdp_tx) },
118 	{ "     xdp redirects", offsetof(struct vmxnet3_rq_driver_stats,
119 					 xdp_redirects) },
120 	{ "     xdp drops", offsetof(struct vmxnet3_rq_driver_stats,
121 				     xdp_drops) },
122 	{ "     xdp aborted", offsetof(struct vmxnet3_rq_driver_stats,
123 				       xdp_aborted) },
124 };
125 
126 /* global stats maintained by the driver */
127 static const struct vmxnet3_stat_desc
128 vmxnet3_global_stats[] = {
129 	/* description,         offset */
130 	{ "tx timeout count",	offsetof(struct vmxnet3_adapter,
131 					 tx_timeout_count) }
132 };
133 
134 
135 void
136 vmxnet3_get_stats64(struct net_device *netdev,
137 		   struct rtnl_link_stats64 *stats)
138 {
139 	struct vmxnet3_adapter *adapter;
140 	struct vmxnet3_tq_driver_stats *drvTxStats;
141 	struct vmxnet3_rq_driver_stats *drvRxStats;
142 	struct UPT1_TxStats *devTxStats;
143 	struct UPT1_RxStats *devRxStats;
144 	unsigned long flags;
145 	int i;
146 
147 	adapter = netdev_priv(netdev);
148 
149 	/* Collect the dev stats into the shared area */
150 	spin_lock_irqsave(&adapter->cmd_lock, flags);
151 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
152 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
153 
154 	for (i = 0; i < adapter->num_tx_queues; i++) {
155 		devTxStats = &adapter->tqd_start[i].stats;
156 		drvTxStats = &adapter->tx_queue[i].stats;
157 		stats->tx_packets += devTxStats->ucastPktsTxOK +
158 				     devTxStats->mcastPktsTxOK +
159 				     devTxStats->bcastPktsTxOK;
160 		stats->tx_bytes += devTxStats->ucastBytesTxOK +
161 				   devTxStats->mcastBytesTxOK +
162 				   devTxStats->bcastBytesTxOK;
163 		stats->tx_errors += devTxStats->pktsTxError;
164 		stats->tx_dropped += drvTxStats->drop_total;
165 	}
166 
167 	for (i = 0; i < adapter->num_rx_queues; i++) {
168 		devRxStats = &adapter->rqd_start[i].stats;
169 		drvRxStats = &adapter->rx_queue[i].stats;
170 		stats->rx_packets += devRxStats->ucastPktsRxOK +
171 				     devRxStats->mcastPktsRxOK +
172 				     devRxStats->bcastPktsRxOK;
173 
174 		stats->rx_bytes += devRxStats->ucastBytesRxOK +
175 				   devRxStats->mcastBytesRxOK +
176 				   devRxStats->bcastBytesRxOK;
177 
178 		stats->rx_errors += devRxStats->pktsRxError;
179 		stats->rx_dropped += drvRxStats->drop_total;
180 		stats->multicast +=  devRxStats->mcastPktsRxOK;
181 	}
182 }
183 
184 static int
185 vmxnet3_get_sset_count(struct net_device *netdev, int sset)
186 {
187 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
188 	switch (sset) {
189 	case ETH_SS_STATS:
190 		return (ARRAY_SIZE(vmxnet3_tq_dev_stats) +
191 			ARRAY_SIZE(vmxnet3_tq_driver_stats)) *
192 		       adapter->num_tx_queues +
193 		       (ARRAY_SIZE(vmxnet3_rq_dev_stats) +
194 			ARRAY_SIZE(vmxnet3_rq_driver_stats)) *
195 		       adapter->num_rx_queues +
196 			ARRAY_SIZE(vmxnet3_global_stats);
197 	default:
198 		return -EOPNOTSUPP;
199 	}
200 }
201 
202 
203 /* This is a version 2 of the vmxnet3 ethtool_regs which goes hand in hand with
204  * the version 2 of the vmxnet3 support for ethtool(8) --register-dump.
205  * Therefore, if any registers are added, removed or modified, then a version
206  * bump and a corresponding change in the vmxnet3 support for ethtool(8)
207  * --register-dump would be required.
208  */
209 static int
210 vmxnet3_get_regs_len(struct net_device *netdev)
211 {
212 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
213 
214 	return ((9 /* BAR1 registers */ +
215 		(1 + adapter->intr.num_intrs) +
216 		(1 + adapter->num_tx_queues * 17 /* Tx queue registers */) +
217 		(1 + adapter->num_rx_queues * 23 /* Rx queue registers */)) *
218 		sizeof(u32));
219 }
220 
221 
222 static void
223 vmxnet3_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
224 {
225 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
226 
227 	strscpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver));
228 
229 	strscpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT,
230 		sizeof(drvinfo->version));
231 
232 	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
233 		sizeof(drvinfo->bus_info));
234 }
235 
236 
237 static void
238 vmxnet3_get_strings(struct net_device *netdev, u32 stringset, u8 *buf)
239 {
240 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
241 	int i, j;
242 
243 	if (stringset != ETH_SS_STATS)
244 		return;
245 
246 	for (j = 0; j < adapter->num_tx_queues; j++) {
247 		for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++)
248 			ethtool_sprintf(&buf, vmxnet3_tq_dev_stats[i].desc);
249 		for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++)
250 			ethtool_sprintf(&buf, vmxnet3_tq_driver_stats[i].desc);
251 	}
252 
253 	for (j = 0; j < adapter->num_rx_queues; j++) {
254 		for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++)
255 			ethtool_sprintf(&buf, vmxnet3_rq_dev_stats[i].desc);
256 		for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++)
257 			ethtool_sprintf(&buf, vmxnet3_rq_driver_stats[i].desc);
258 	}
259 
260 	for (i = 0; i < ARRAY_SIZE(vmxnet3_global_stats); i++)
261 		ethtool_sprintf(&buf, vmxnet3_global_stats[i].desc);
262 }
263 
264 netdev_features_t vmxnet3_fix_features(struct net_device *netdev,
265 				       netdev_features_t features)
266 {
267 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
268 
269 	/* If Rx checksum is disabled, then LRO should also be disabled */
270 	if (!(features & NETIF_F_RXCSUM))
271 		features &= ~NETIF_F_LRO;
272 
273 	/* If XDP is enabled, then LRO should not be enabled */
274 	if (vmxnet3_xdp_enabled(adapter) && (features & NETIF_F_LRO)) {
275 		netdev_err(netdev, "LRO is not supported with XDP");
276 		features &= ~NETIF_F_LRO;
277 	}
278 
279 	return features;
280 }
281 
282 netdev_features_t vmxnet3_features_check(struct sk_buff *skb,
283 					 struct net_device *netdev,
284 					 netdev_features_t features)
285 {
286 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
287 
288 	/* Validate if the tunneled packet is being offloaded by the device */
289 	if (VMXNET3_VERSION_GE_4(adapter) &&
290 	    skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) {
291 		u8 l4_proto = 0;
292 		u16 port;
293 		struct udphdr *udph;
294 
295 		switch (vlan_get_protocol(skb)) {
296 		case htons(ETH_P_IP):
297 			l4_proto = ip_hdr(skb)->protocol;
298 			break;
299 		case htons(ETH_P_IPV6):
300 			l4_proto = ipv6_hdr(skb)->nexthdr;
301 			break;
302 		default:
303 			return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
304 		}
305 
306 		switch (l4_proto) {
307 		case IPPROTO_UDP:
308 			udph = udp_hdr(skb);
309 			port = be16_to_cpu(udph->dest);
310 			/* Check if offloaded port is supported */
311 			if (port != GENEVE_UDP_PORT &&
312 			    port != IANA_VXLAN_UDP_PORT &&
313 			    port != VXLAN_UDP_PORT) {
314 				return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
315 			}
316 			break;
317 		default:
318 			return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
319 		}
320 	}
321 	return features;
322 }
323 
324 static void vmxnet3_enable_encap_offloads(struct net_device *netdev, netdev_features_t features)
325 {
326 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
327 
328 	if (VMXNET3_VERSION_GE_4(adapter)) {
329 		netdev->hw_enc_features |= NETIF_F_SG | NETIF_F_RXCSUM |
330 			NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
331 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
332 			NETIF_F_LRO;
333 		if (features & NETIF_F_GSO_UDP_TUNNEL)
334 			netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
335 		if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
336 			netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
337 	}
338 	if (VMXNET3_VERSION_GE_7(adapter)) {
339 		unsigned long flags;
340 
341 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
342 					       VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) {
343 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD;
344 		}
345 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
346 					       VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) {
347 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD;
348 		}
349 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
350 					       VMXNET3_CAP_GENEVE_TSO)) {
351 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO;
352 		}
353 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
354 					       VMXNET3_CAP_VXLAN_TSO)) {
355 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO;
356 		}
357 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
358 					       VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) {
359 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD;
360 		}
361 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
362 					       VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) {
363 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD;
364 		}
365 
366 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
367 		spin_lock_irqsave(&adapter->cmd_lock, flags);
368 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
369 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
370 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
371 
372 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) &&
373 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) &&
374 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) &&
375 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) {
376 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL;
377 		}
378 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) &&
379 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) {
380 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
381 		}
382 	}
383 }
384 
385 static void vmxnet3_disable_encap_offloads(struct net_device *netdev)
386 {
387 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
388 
389 	if (VMXNET3_VERSION_GE_4(adapter)) {
390 		netdev->hw_enc_features &= ~(NETIF_F_SG | NETIF_F_RXCSUM |
391 			NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
392 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
393 			NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL |
394 			NETIF_F_GSO_UDP_TUNNEL_CSUM);
395 	}
396 	if (VMXNET3_VERSION_GE_7(adapter)) {
397 		unsigned long flags;
398 
399 		adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD |
400 					  1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD  |
401 					  1UL << VMXNET3_CAP_GENEVE_TSO |
402 					  1UL << VMXNET3_CAP_VXLAN_TSO  |
403 					  1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD |
404 					  1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD);
405 
406 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
407 		spin_lock_irqsave(&adapter->cmd_lock, flags);
408 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
409 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
410 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
411 	}
412 }
413 
414 int vmxnet3_set_features(struct net_device *netdev, netdev_features_t features)
415 {
416 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
417 	unsigned long flags;
418 	netdev_features_t changed = features ^ netdev->features;
419 	netdev_features_t tun_offload_mask = NETIF_F_GSO_UDP_TUNNEL |
420 					     NETIF_F_GSO_UDP_TUNNEL_CSUM;
421 	u8 udp_tun_enabled = (netdev->features & tun_offload_mask) != 0;
422 
423 	if (changed & (NETIF_F_RXCSUM | NETIF_F_LRO |
424 		       NETIF_F_HW_VLAN_CTAG_RX | tun_offload_mask)) {
425 		if (features & NETIF_F_RXCSUM)
426 			adapter->shared->devRead.misc.uptFeatures |=
427 			UPT1_F_RXCSUM;
428 		else
429 			adapter->shared->devRead.misc.uptFeatures &=
430 			~UPT1_F_RXCSUM;
431 
432 		/* update hardware LRO capability accordingly */
433 		if (features & NETIF_F_LRO)
434 			adapter->shared->devRead.misc.uptFeatures |=
435 							UPT1_F_LRO;
436 		else
437 			adapter->shared->devRead.misc.uptFeatures &=
438 							~UPT1_F_LRO;
439 
440 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
441 			adapter->shared->devRead.misc.uptFeatures |=
442 			UPT1_F_RXVLAN;
443 		else
444 			adapter->shared->devRead.misc.uptFeatures &=
445 			~UPT1_F_RXVLAN;
446 
447 		if ((features & tun_offload_mask) != 0) {
448 			vmxnet3_enable_encap_offloads(netdev, features);
449 			adapter->shared->devRead.misc.uptFeatures |=
450 			UPT1_F_RXINNEROFLD;
451 		} else if ((features & tun_offload_mask) == 0 &&
452 			   udp_tun_enabled) {
453 			vmxnet3_disable_encap_offloads(netdev);
454 			adapter->shared->devRead.misc.uptFeatures &=
455 			~UPT1_F_RXINNEROFLD;
456 		}
457 
458 		spin_lock_irqsave(&adapter->cmd_lock, flags);
459 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
460 				       VMXNET3_CMD_UPDATE_FEATURE);
461 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
462 	}
463 	return 0;
464 }
465 
466 static void
467 vmxnet3_get_ethtool_stats(struct net_device *netdev,
468 			  struct ethtool_stats *stats, u64  *buf)
469 {
470 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
471 	unsigned long flags;
472 	u8 *base;
473 	int i;
474 	int j = 0;
475 
476 	spin_lock_irqsave(&adapter->cmd_lock, flags);
477 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
478 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
479 
480 	/* this does assume each counter is 64-bit wide */
481 	for (j = 0; j < adapter->num_tx_queues; j++) {
482 		base = (u8 *)&adapter->tqd_start[j].stats;
483 		*buf++ = (u64)j;
484 		for (i = 1; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++)
485 			*buf++ = *(u64 *)(base +
486 					  vmxnet3_tq_dev_stats[i].offset);
487 
488 		base = (u8 *)&adapter->tx_queue[j].stats;
489 		for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++)
490 			*buf++ = *(u64 *)(base +
491 					  vmxnet3_tq_driver_stats[i].offset);
492 	}
493 
494 	for (j = 0; j < adapter->num_rx_queues; j++) {
495 		base = (u8 *)&adapter->rqd_start[j].stats;
496 		*buf++ = (u64) j;
497 		for (i = 1; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++)
498 			*buf++ = *(u64 *)(base +
499 					  vmxnet3_rq_dev_stats[i].offset);
500 
501 		base = (u8 *)&adapter->rx_queue[j].stats;
502 		for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++)
503 			*buf++ = *(u64 *)(base +
504 					  vmxnet3_rq_driver_stats[i].offset);
505 	}
506 
507 	base = (u8 *)adapter;
508 	for (i = 0; i < ARRAY_SIZE(vmxnet3_global_stats); i++)
509 		*buf++ = *(u64 *)(base + vmxnet3_global_stats[i].offset);
510 }
511 
512 
513 /* This is a version 2 of the vmxnet3 ethtool_regs which goes hand in hand with
514  * the version 2 of the vmxnet3 support for ethtool(8) --register-dump.
515  * Therefore, if any registers are added, removed or modified, then a version
516  * bump and a corresponding change in the vmxnet3 support for ethtool(8)
517  * --register-dump would be required.
518  */
519 static void
520 vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
521 {
522 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
523 	u32 *buf = p;
524 	int i = 0, j = 0;
525 
526 	memset(p, 0, vmxnet3_get_regs_len(netdev));
527 
528 	regs->version = 2;
529 
530 	/* Update vmxnet3_get_regs_len if we want to dump more registers */
531 
532 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
533 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
534 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DSAL);
535 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DSAH);
536 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
537 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
538 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
539 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
540 	buf[j++] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ECR);
541 
542 	buf[j++] = adapter->intr.num_intrs;
543 	for (i = 0; i < adapter->intr.num_intrs; i++) {
544 		buf[j++] = VMXNET3_READ_BAR0_REG(adapter, VMXNET3_REG_IMR
545 						 + i * VMXNET3_REG_ALIGN);
546 	}
547 
548 	buf[j++] = adapter->num_tx_queues;
549 	for (i = 0; i < adapter->num_tx_queues; i++) {
550 		struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];
551 
552 		buf[j++] = VMXNET3_READ_BAR0_REG(adapter, adapter->tx_prod_offset +
553 						 i * VMXNET3_REG_ALIGN);
554 
555 		buf[j++] = VMXNET3_GET_ADDR_LO(tq->tx_ring.basePA);
556 		buf[j++] = VMXNET3_GET_ADDR_HI(tq->tx_ring.basePA);
557 		buf[j++] = tq->tx_ring.size;
558 		buf[j++] = tq->tx_ring.next2fill;
559 		buf[j++] = tq->tx_ring.next2comp;
560 		buf[j++] = tq->tx_ring.gen;
561 
562 		buf[j++] = VMXNET3_GET_ADDR_LO(tq->data_ring.basePA);
563 		buf[j++] = VMXNET3_GET_ADDR_HI(tq->data_ring.basePA);
564 		buf[j++] = tq->data_ring.size;
565 		buf[j++] = tq->txdata_desc_size;
566 
567 		buf[j++] = VMXNET3_GET_ADDR_LO(tq->comp_ring.basePA);
568 		buf[j++] = VMXNET3_GET_ADDR_HI(tq->comp_ring.basePA);
569 		buf[j++] = tq->comp_ring.size;
570 		buf[j++] = tq->comp_ring.next2proc;
571 		buf[j++] = tq->comp_ring.gen;
572 
573 		buf[j++] = tq->stopped;
574 	}
575 
576 	buf[j++] = adapter->num_rx_queues;
577 	for (i = 0; i < adapter->num_rx_queues; i++) {
578 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
579 
580 		buf[j++] =  VMXNET3_READ_BAR0_REG(adapter, adapter->rx_prod_offset +
581 						  i * VMXNET3_REG_ALIGN);
582 		buf[j++] =  VMXNET3_READ_BAR0_REG(adapter, adapter->rx_prod2_offset +
583 						  i * VMXNET3_REG_ALIGN);
584 
585 		buf[j++] = VMXNET3_GET_ADDR_LO(rq->rx_ring[0].basePA);
586 		buf[j++] = VMXNET3_GET_ADDR_HI(rq->rx_ring[0].basePA);
587 		buf[j++] = rq->rx_ring[0].size;
588 		buf[j++] = rq->rx_ring[0].next2fill;
589 		buf[j++] = rq->rx_ring[0].next2comp;
590 		buf[j++] = rq->rx_ring[0].gen;
591 
592 		buf[j++] = VMXNET3_GET_ADDR_LO(rq->rx_ring[1].basePA);
593 		buf[j++] = VMXNET3_GET_ADDR_HI(rq->rx_ring[1].basePA);
594 		buf[j++] = rq->rx_ring[1].size;
595 		buf[j++] = rq->rx_ring[1].next2fill;
596 		buf[j++] = rq->rx_ring[1].next2comp;
597 		buf[j++] = rq->rx_ring[1].gen;
598 
599 		buf[j++] = VMXNET3_GET_ADDR_LO(rq->data_ring.basePA);
600 		buf[j++] = VMXNET3_GET_ADDR_HI(rq->data_ring.basePA);
601 		buf[j++] = rq->rx_ring[0].size;
602 		buf[j++] = rq->data_ring.desc_size;
603 
604 		buf[j++] = VMXNET3_GET_ADDR_LO(rq->comp_ring.basePA);
605 		buf[j++] = VMXNET3_GET_ADDR_HI(rq->comp_ring.basePA);
606 		buf[j++] = rq->comp_ring.size;
607 		buf[j++] = rq->comp_ring.next2proc;
608 		buf[j++] = rq->comp_ring.gen;
609 	}
610 }
611 
612 
613 static void
614 vmxnet3_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
615 {
616 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
617 
618 	wol->supported = WAKE_UCAST | WAKE_ARP | WAKE_MAGIC;
619 	wol->wolopts = adapter->wol;
620 }
621 
622 
623 static int
624 vmxnet3_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
625 {
626 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
627 
628 	if (wol->wolopts & (WAKE_PHY | WAKE_MCAST | WAKE_BCAST |
629 			    WAKE_MAGICSECURE)) {
630 		return -EOPNOTSUPP;
631 	}
632 
633 	adapter->wol = wol->wolopts;
634 
635 	device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
636 
637 	return 0;
638 }
639 
640 
641 static int
642 vmxnet3_get_link_ksettings(struct net_device *netdev,
643 			   struct ethtool_link_ksettings *ecmd)
644 {
645 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
646 
647 	ethtool_link_ksettings_zero_link_mode(ecmd, supported);
648 	ethtool_link_ksettings_add_link_mode(ecmd, supported, 10000baseT_Full);
649 	ethtool_link_ksettings_add_link_mode(ecmd, supported, 1000baseT_Full);
650 	ethtool_link_ksettings_add_link_mode(ecmd, supported, TP);
651 	ethtool_link_ksettings_zero_link_mode(ecmd, advertising);
652 	ethtool_link_ksettings_add_link_mode(ecmd, advertising, TP);
653 	ecmd->base.port = PORT_TP;
654 
655 	if (adapter->link_speed) {
656 		ecmd->base.speed = adapter->link_speed;
657 		ecmd->base.duplex = DUPLEX_FULL;
658 	} else {
659 		ecmd->base.speed = SPEED_UNKNOWN;
660 		ecmd->base.duplex = DUPLEX_UNKNOWN;
661 	}
662 	return 0;
663 }
664 
665 static void
666 vmxnet3_get_ringparam(struct net_device *netdev,
667 		      struct ethtool_ringparam *param,
668 		      struct kernel_ethtool_ringparam *kernel_param,
669 		      struct netlink_ext_ack *extack)
670 {
671 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
672 
673 	param->rx_max_pending = VMXNET3_RX_RING_MAX_SIZE;
674 	param->tx_max_pending = VMXNET3_TX_RING_MAX_SIZE;
675 	param->rx_mini_max_pending = VMXNET3_VERSION_GE_3(adapter) ?
676 		VMXNET3_RXDATA_DESC_MAX_SIZE : 0;
677 	param->rx_jumbo_max_pending = VMXNET3_RX_RING2_MAX_SIZE;
678 
679 	param->rx_pending = adapter->rx_ring_size;
680 	param->tx_pending = adapter->tx_ring_size;
681 	param->rx_mini_pending = VMXNET3_VERSION_GE_3(adapter) ?
682 		adapter->rxdata_desc_size : 0;
683 	param->rx_jumbo_pending = adapter->rx_ring2_size;
684 }
685 
686 static int
687 vmxnet3_set_ringparam(struct net_device *netdev,
688 		      struct ethtool_ringparam *param,
689 		      struct kernel_ethtool_ringparam *kernel_param,
690 		      struct netlink_ext_ack *extack)
691 {
692 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
693 	u32 new_tx_ring_size, new_rx_ring_size, new_rx_ring2_size;
694 	u16 new_rxdata_desc_size;
695 	u32 sz;
696 	int err = 0;
697 
698 	if (param->tx_pending == 0 || param->tx_pending >
699 						VMXNET3_TX_RING_MAX_SIZE)
700 		return -EINVAL;
701 
702 	if (param->rx_pending == 0 || param->rx_pending >
703 						VMXNET3_RX_RING_MAX_SIZE)
704 		return -EINVAL;
705 
706 	if (param->rx_jumbo_pending == 0 ||
707 	    param->rx_jumbo_pending > VMXNET3_RX_RING2_MAX_SIZE)
708 		return -EINVAL;
709 
710 	/* if adapter not yet initialized, do nothing */
711 	if (adapter->rx_buf_per_pkt == 0) {
712 		netdev_err(netdev, "adapter not completely initialized, "
713 			   "ring size cannot be changed yet\n");
714 		return -EOPNOTSUPP;
715 	}
716 
717 	if (VMXNET3_VERSION_GE_3(adapter)) {
718 		if (param->rx_mini_pending > VMXNET3_RXDATA_DESC_MAX_SIZE)
719 			return -EINVAL;
720 	} else if (param->rx_mini_pending != 0) {
721 		return -EINVAL;
722 	}
723 
724 	/* round it up to a multiple of VMXNET3_RING_SIZE_ALIGN */
725 	new_tx_ring_size = (param->tx_pending + VMXNET3_RING_SIZE_MASK) &
726 							~VMXNET3_RING_SIZE_MASK;
727 	new_tx_ring_size = min_t(u32, new_tx_ring_size,
728 				 VMXNET3_TX_RING_MAX_SIZE);
729 	if (new_tx_ring_size > VMXNET3_TX_RING_MAX_SIZE || (new_tx_ring_size %
730 						VMXNET3_RING_SIZE_ALIGN) != 0)
731 		return -EINVAL;
732 
733 	/* ring0 has to be a multiple of
734 	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
735 	 */
736 	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
737 	new_rx_ring_size = (param->rx_pending + sz - 1) / sz * sz;
738 	new_rx_ring_size = min_t(u32, new_rx_ring_size,
739 				 VMXNET3_RX_RING_MAX_SIZE / sz * sz);
740 	if (new_rx_ring_size > VMXNET3_RX_RING_MAX_SIZE || (new_rx_ring_size %
741 							   sz) != 0)
742 		return -EINVAL;
743 
744 	/* ring2 has to be a multiple of VMXNET3_RING_SIZE_ALIGN */
745 	new_rx_ring2_size = (param->rx_jumbo_pending + VMXNET3_RING_SIZE_MASK) &
746 				~VMXNET3_RING_SIZE_MASK;
747 	new_rx_ring2_size = min_t(u32, new_rx_ring2_size,
748 				  VMXNET3_RX_RING2_MAX_SIZE);
749 
750 	/* For v7 and later, keep ring size power of 2 for UPT */
751 	if (VMXNET3_VERSION_GE_7(adapter)) {
752 		new_tx_ring_size = rounddown_pow_of_two(new_tx_ring_size);
753 		new_rx_ring_size = rounddown_pow_of_two(new_rx_ring_size);
754 		new_rx_ring2_size = rounddown_pow_of_two(new_rx_ring2_size);
755 	}
756 
757 	/* rx data ring buffer size has to be a multiple of
758 	 * VMXNET3_RXDATA_DESC_SIZE_ALIGN
759 	 */
760 	new_rxdata_desc_size =
761 		(param->rx_mini_pending + VMXNET3_RXDATA_DESC_SIZE_MASK) &
762 		~VMXNET3_RXDATA_DESC_SIZE_MASK;
763 	new_rxdata_desc_size = min_t(u16, new_rxdata_desc_size,
764 				     VMXNET3_RXDATA_DESC_MAX_SIZE);
765 
766 	if (new_tx_ring_size == adapter->tx_ring_size &&
767 	    new_rx_ring_size == adapter->rx_ring_size &&
768 	    new_rx_ring2_size == adapter->rx_ring2_size &&
769 	    new_rxdata_desc_size == adapter->rxdata_desc_size) {
770 		return 0;
771 	}
772 
773 	/*
774 	 * Reset_work may be in the middle of resetting the device, wait for its
775 	 * completion.
776 	 */
777 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
778 		usleep_range(1000, 2000);
779 
780 	if (netif_running(netdev)) {
781 		vmxnet3_quiesce_dev(adapter);
782 		vmxnet3_reset_dev(adapter);
783 
784 		/* recreate the rx queue and the tx queue based on the
785 		 * new sizes */
786 		vmxnet3_tq_destroy_all(adapter);
787 		vmxnet3_rq_destroy_all(adapter);
788 
789 		err = vmxnet3_create_queues(adapter, new_tx_ring_size,
790 					    new_rx_ring_size, new_rx_ring2_size,
791 					    adapter->txdata_desc_size,
792 					    new_rxdata_desc_size);
793 		if (err) {
794 			/* failed, most likely because of OOM, try default
795 			 * size */
796 			netdev_err(netdev, "failed to apply new sizes, "
797 				   "try the default ones\n");
798 			new_rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
799 			new_rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
800 			new_tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
801 			new_rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
802 				VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
803 
804 			err = vmxnet3_create_queues(adapter,
805 						    new_tx_ring_size,
806 						    new_rx_ring_size,
807 						    new_rx_ring2_size,
808 						    adapter->txdata_desc_size,
809 						    new_rxdata_desc_size);
810 			if (err) {
811 				netdev_err(netdev, "failed to create queues "
812 					   "with default sizes. Closing it\n");
813 				goto out;
814 			}
815 		}
816 
817 		err = vmxnet3_activate_dev(adapter);
818 		if (err)
819 			netdev_err(netdev, "failed to re-activate, error %d."
820 				   " Closing it\n", err);
821 	}
822 	adapter->tx_ring_size = new_tx_ring_size;
823 	adapter->rx_ring_size = new_rx_ring_size;
824 	adapter->rx_ring2_size = new_rx_ring2_size;
825 	adapter->rxdata_desc_size = new_rxdata_desc_size;
826 
827 out:
828 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
829 	if (err)
830 		vmxnet3_force_close(adapter);
831 
832 	return err;
833 }
834 
835 static int
836 vmxnet3_get_rss_hash_opts(struct vmxnet3_adapter *adapter,
837 			  struct ethtool_rxnfc *info)
838 {
839 	enum Vmxnet3_RSSField rss_fields;
840 
841 	if (netif_running(adapter->netdev)) {
842 		unsigned long flags;
843 
844 		spin_lock_irqsave(&adapter->cmd_lock, flags);
845 
846 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
847 				       VMXNET3_CMD_GET_RSS_FIELDS);
848 		rss_fields = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
849 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
850 	} else {
851 		rss_fields = adapter->rss_fields;
852 	}
853 
854 	info->data = 0;
855 
856 	/* Report default options for RSS on vmxnet3 */
857 	switch (info->flow_type) {
858 	case TCP_V4_FLOW:
859 	case TCP_V6_FLOW:
860 		info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3 |
861 			      RXH_IP_SRC | RXH_IP_DST;
862 		break;
863 	case UDP_V4_FLOW:
864 		if (rss_fields & VMXNET3_RSS_FIELDS_UDPIP4)
865 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
866 		info->data |= RXH_IP_SRC | RXH_IP_DST;
867 		break;
868 	case AH_ESP_V4_FLOW:
869 	case AH_V4_FLOW:
870 	case ESP_V4_FLOW:
871 		if (rss_fields & VMXNET3_RSS_FIELDS_ESPIP4)
872 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
873 		fallthrough;
874 	case SCTP_V4_FLOW:
875 	case IPV4_FLOW:
876 		info->data |= RXH_IP_SRC | RXH_IP_DST;
877 		break;
878 	case UDP_V6_FLOW:
879 		if (rss_fields & VMXNET3_RSS_FIELDS_UDPIP6)
880 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
881 		info->data |= RXH_IP_SRC | RXH_IP_DST;
882 		break;
883 	case AH_ESP_V6_FLOW:
884 	case AH_V6_FLOW:
885 	case ESP_V6_FLOW:
886 		if (VMXNET3_VERSION_GE_6(adapter) &&
887 		    (rss_fields & VMXNET3_RSS_FIELDS_ESPIP6))
888 			info->data |= RXH_L4_B_0_1 | RXH_L4_B_2_3;
889 		fallthrough;
890 	case SCTP_V6_FLOW:
891 	case IPV6_FLOW:
892 		info->data |= RXH_IP_SRC | RXH_IP_DST;
893 		break;
894 	default:
895 		return -EINVAL;
896 	}
897 
898 	return 0;
899 }
900 
901 static int
902 vmxnet3_set_rss_hash_opt(struct net_device *netdev,
903 			 struct vmxnet3_adapter *adapter,
904 			 struct ethtool_rxnfc *nfc)
905 {
906 	enum Vmxnet3_RSSField rss_fields = adapter->rss_fields;
907 
908 	/* RSS does not support anything other than hashing
909 	 * to queues on src and dst IPs and ports
910 	 */
911 	if (nfc->data & ~(RXH_IP_SRC | RXH_IP_DST |
912 			  RXH_L4_B_0_1 | RXH_L4_B_2_3))
913 		return -EINVAL;
914 
915 	switch (nfc->flow_type) {
916 	case TCP_V4_FLOW:
917 	case TCP_V6_FLOW:
918 		if (!(nfc->data & RXH_IP_SRC) ||
919 		    !(nfc->data & RXH_IP_DST) ||
920 		    !(nfc->data & RXH_L4_B_0_1) ||
921 		    !(nfc->data & RXH_L4_B_2_3))
922 			return -EINVAL;
923 		break;
924 	case UDP_V4_FLOW:
925 		if (!(nfc->data & RXH_IP_SRC) ||
926 		    !(nfc->data & RXH_IP_DST))
927 			return -EINVAL;
928 		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
929 		case 0:
930 			rss_fields &= ~VMXNET3_RSS_FIELDS_UDPIP4;
931 			break;
932 		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
933 			rss_fields |= VMXNET3_RSS_FIELDS_UDPIP4;
934 			break;
935 		default:
936 			return -EINVAL;
937 		}
938 		break;
939 	case UDP_V6_FLOW:
940 		if (!(nfc->data & RXH_IP_SRC) ||
941 		    !(nfc->data & RXH_IP_DST))
942 			return -EINVAL;
943 		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
944 		case 0:
945 			rss_fields &= ~VMXNET3_RSS_FIELDS_UDPIP6;
946 			break;
947 		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
948 			rss_fields |= VMXNET3_RSS_FIELDS_UDPIP6;
949 			break;
950 		default:
951 			return -EINVAL;
952 		}
953 		break;
954 	case ESP_V4_FLOW:
955 	case AH_V4_FLOW:
956 	case AH_ESP_V4_FLOW:
957 		if (!(nfc->data & RXH_IP_SRC) ||
958 		    !(nfc->data & RXH_IP_DST))
959 			return -EINVAL;
960 		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
961 		case 0:
962 			rss_fields &= ~VMXNET3_RSS_FIELDS_ESPIP4;
963 			break;
964 		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
965 			rss_fields |= VMXNET3_RSS_FIELDS_ESPIP4;
966 		break;
967 		default:
968 			return -EINVAL;
969 		}
970 		break;
971 	case ESP_V6_FLOW:
972 	case AH_V6_FLOW:
973 	case AH_ESP_V6_FLOW:
974 		if (!VMXNET3_VERSION_GE_6(adapter))
975 			return -EOPNOTSUPP;
976 		if (!(nfc->data & RXH_IP_SRC) ||
977 		    !(nfc->data & RXH_IP_DST))
978 			return -EINVAL;
979 		switch (nfc->data & (RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
980 		case 0:
981 			rss_fields &= ~VMXNET3_RSS_FIELDS_ESPIP6;
982 			break;
983 		case (RXH_L4_B_0_1 | RXH_L4_B_2_3):
984 			rss_fields |= VMXNET3_RSS_FIELDS_ESPIP6;
985 			break;
986 		default:
987 			return -EINVAL;
988 		}
989 		break;
990 	case SCTP_V4_FLOW:
991 	case SCTP_V6_FLOW:
992 		if (!(nfc->data & RXH_IP_SRC) ||
993 		    !(nfc->data & RXH_IP_DST) ||
994 		    (nfc->data & RXH_L4_B_0_1) ||
995 		    (nfc->data & RXH_L4_B_2_3))
996 			return -EINVAL;
997 		break;
998 	default:
999 		return -EINVAL;
1000 	}
1001 
1002 	/* if we changed something we need to update flags */
1003 	if (rss_fields != adapter->rss_fields) {
1004 		adapter->default_rss_fields = false;
1005 		if (netif_running(netdev)) {
1006 			struct Vmxnet3_DriverShared *shared = adapter->shared;
1007 			union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
1008 			unsigned long flags;
1009 
1010 			if (VMXNET3_VERSION_GE_7(adapter)) {
1011 				if ((rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 ||
1012 				     rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) &&
1013 				    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
1014 							       VMXNET3_CAP_UDP_RSS)) {
1015 					adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS;
1016 				} else {
1017 					adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS);
1018 				}
1019 				if ((rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) &&
1020 				    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
1021 							       VMXNET3_CAP_ESP_RSS_IPV4)) {
1022 					adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4;
1023 				} else {
1024 					adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4);
1025 				}
1026 				if ((rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) &&
1027 				    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
1028 							       VMXNET3_CAP_ESP_RSS_IPV6)) {
1029 					adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6;
1030 				} else {
1031 					adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6);
1032 				}
1033 
1034 				VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR,
1035 						       adapter->dev_caps[0]);
1036 				spin_lock_irqsave(&adapter->cmd_lock, flags);
1037 				VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1038 						       VMXNET3_CMD_GET_DCR0_REG);
1039 				adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter,
1040 									     VMXNET3_REG_CMD);
1041 				spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1042 			}
1043 			spin_lock_irqsave(&adapter->cmd_lock, flags);
1044 			cmdInfo->setRssFields = rss_fields;
1045 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1046 					       VMXNET3_CMD_SET_RSS_FIELDS);
1047 
1048 			/* Not all requested RSS may get applied, so get and
1049 			 * cache what was actually applied.
1050 			 */
1051 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1052 					       VMXNET3_CMD_GET_RSS_FIELDS);
1053 			adapter->rss_fields =
1054 				VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
1055 			spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1056 		} else {
1057 			/* When the device is activated, we will try to apply
1058 			 * these rules and cache the applied value later.
1059 			 */
1060 			adapter->rss_fields = rss_fields;
1061 		}
1062 	}
1063 	return 0;
1064 }
1065 
1066 static int
1067 vmxnet3_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
1068 		  u32 *rules)
1069 {
1070 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1071 	int err = 0;
1072 
1073 	switch (info->cmd) {
1074 	case ETHTOOL_GRXRINGS:
1075 		info->data = adapter->num_rx_queues;
1076 		break;
1077 	case ETHTOOL_GRXFH:
1078 		if (!VMXNET3_VERSION_GE_4(adapter)) {
1079 			err = -EOPNOTSUPP;
1080 			break;
1081 		}
1082 #ifdef VMXNET3_RSS
1083 		if (!adapter->rss) {
1084 			err = -EOPNOTSUPP;
1085 			break;
1086 		}
1087 #endif
1088 		err = vmxnet3_get_rss_hash_opts(adapter, info);
1089 		break;
1090 	default:
1091 		err = -EOPNOTSUPP;
1092 		break;
1093 	}
1094 
1095 	return err;
1096 }
1097 
1098 static int
1099 vmxnet3_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info)
1100 {
1101 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1102 	int err = 0;
1103 
1104 	if (!VMXNET3_VERSION_GE_4(adapter)) {
1105 		err = -EOPNOTSUPP;
1106 		goto done;
1107 	}
1108 #ifdef VMXNET3_RSS
1109 	if (!adapter->rss) {
1110 		err = -EOPNOTSUPP;
1111 		goto done;
1112 	}
1113 #endif
1114 
1115 	switch (info->cmd) {
1116 	case ETHTOOL_SRXFH:
1117 		err = vmxnet3_set_rss_hash_opt(netdev, adapter, info);
1118 		break;
1119 	default:
1120 		err = -EOPNOTSUPP;
1121 		break;
1122 	}
1123 
1124 done:
1125 	return err;
1126 }
1127 
1128 #ifdef VMXNET3_RSS
1129 static u32
1130 vmxnet3_get_rss_indir_size(struct net_device *netdev)
1131 {
1132 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1133 	struct UPT1_RSSConf *rssConf = adapter->rss_conf;
1134 
1135 	return rssConf->indTableSize;
1136 }
1137 
1138 static int
1139 vmxnet3_get_rss(struct net_device *netdev, u32 *p, u8 *key, u8 *hfunc)
1140 {
1141 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1142 	struct UPT1_RSSConf *rssConf = adapter->rss_conf;
1143 	unsigned int n = rssConf->indTableSize;
1144 
1145 	if (hfunc)
1146 		*hfunc = ETH_RSS_HASH_TOP;
1147 	if (!p)
1148 		return 0;
1149 	if (n > UPT1_RSS_MAX_IND_TABLE_SIZE)
1150 		return 0;
1151 	while (n--)
1152 		p[n] = rssConf->indTable[n];
1153 	return 0;
1154 
1155 }
1156 
1157 static int
1158 vmxnet3_set_rss(struct net_device *netdev, const u32 *p, const u8 *key,
1159 		const u8 hfunc)
1160 {
1161 	unsigned int i;
1162 	unsigned long flags;
1163 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1164 	struct UPT1_RSSConf *rssConf = adapter->rss_conf;
1165 
1166 	/* We do not allow change in unsupported parameters */
1167 	if (key ||
1168 	    (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
1169 		return -EOPNOTSUPP;
1170 	if (!p)
1171 		return 0;
1172 	for (i = 0; i < rssConf->indTableSize; i++)
1173 		rssConf->indTable[i] = p[i];
1174 
1175 	spin_lock_irqsave(&adapter->cmd_lock, flags);
1176 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1177 			       VMXNET3_CMD_UPDATE_RSSIDT);
1178 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1179 
1180 	return 0;
1181 
1182 }
1183 #endif
1184 
1185 static int vmxnet3_get_coalesce(struct net_device *netdev,
1186 				struct ethtool_coalesce *ec,
1187 				struct kernel_ethtool_coalesce *kernel_coal,
1188 				struct netlink_ext_ack *extack)
1189 {
1190 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1191 
1192 	if (!VMXNET3_VERSION_GE_3(adapter))
1193 		return -EOPNOTSUPP;
1194 
1195 	switch (adapter->coal_conf->coalMode) {
1196 	case VMXNET3_COALESCE_DISABLED:
1197 		/* struct ethtool_coalesce is already initialized to 0 */
1198 		break;
1199 	case VMXNET3_COALESCE_ADAPT:
1200 		ec->use_adaptive_rx_coalesce = true;
1201 		break;
1202 	case VMXNET3_COALESCE_STATIC:
1203 		ec->tx_max_coalesced_frames =
1204 			adapter->coal_conf->coalPara.coalStatic.tx_comp_depth;
1205 		ec->rx_max_coalesced_frames =
1206 			adapter->coal_conf->coalPara.coalStatic.rx_depth;
1207 		break;
1208 	case VMXNET3_COALESCE_RBC: {
1209 		u32 rbc_rate;
1210 
1211 		rbc_rate = adapter->coal_conf->coalPara.coalRbc.rbc_rate;
1212 		ec->rx_coalesce_usecs = VMXNET3_COAL_RBC_USECS(rbc_rate);
1213 	}
1214 		break;
1215 	default:
1216 		return -EOPNOTSUPP;
1217 	}
1218 
1219 	return 0;
1220 }
1221 
1222 static int vmxnet3_set_coalesce(struct net_device *netdev,
1223 				struct ethtool_coalesce *ec,
1224 				struct kernel_ethtool_coalesce *kernel_coal,
1225 				struct netlink_ext_ack *extack)
1226 {
1227 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1228 	struct Vmxnet3_DriverShared *shared = adapter->shared;
1229 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
1230 	unsigned long flags;
1231 
1232 	if (!VMXNET3_VERSION_GE_3(adapter))
1233 		return -EOPNOTSUPP;
1234 
1235 	if ((ec->rx_coalesce_usecs == 0) &&
1236 	    (ec->use_adaptive_rx_coalesce == 0) &&
1237 	    (ec->tx_max_coalesced_frames == 0) &&
1238 	    (ec->rx_max_coalesced_frames == 0)) {
1239 		memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
1240 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
1241 		goto done;
1242 	}
1243 
1244 	if (ec->rx_coalesce_usecs != 0) {
1245 		u32 rbc_rate;
1246 
1247 		if ((ec->use_adaptive_rx_coalesce != 0) ||
1248 		    (ec->tx_max_coalesced_frames != 0) ||
1249 		    (ec->rx_max_coalesced_frames != 0)) {
1250 			return -EINVAL;
1251 		}
1252 
1253 		rbc_rate = VMXNET3_COAL_RBC_RATE(ec->rx_coalesce_usecs);
1254 		if (rbc_rate < VMXNET3_COAL_RBC_MIN_RATE ||
1255 		    rbc_rate > VMXNET3_COAL_RBC_MAX_RATE) {
1256 			return -EINVAL;
1257 		}
1258 
1259 		memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
1260 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_RBC;
1261 		adapter->coal_conf->coalPara.coalRbc.rbc_rate = rbc_rate;
1262 		goto done;
1263 	}
1264 
1265 	if (ec->use_adaptive_rx_coalesce != 0) {
1266 		if (ec->tx_max_coalesced_frames != 0 ||
1267 		    ec->rx_max_coalesced_frames != 0) {
1268 			return -EINVAL;
1269 		}
1270 		memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
1271 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_ADAPT;
1272 		goto done;
1273 	}
1274 
1275 	if ((ec->tx_max_coalesced_frames != 0) ||
1276 	    (ec->rx_max_coalesced_frames != 0)) {
1277 		if ((ec->tx_max_coalesced_frames >
1278 		    VMXNET3_COAL_STATIC_MAX_DEPTH) ||
1279 		    (ec->rx_max_coalesced_frames >
1280 		     VMXNET3_COAL_STATIC_MAX_DEPTH)) {
1281 			return -EINVAL;
1282 		}
1283 
1284 		memset(adapter->coal_conf, 0, sizeof(*adapter->coal_conf));
1285 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_STATIC;
1286 
1287 		adapter->coal_conf->coalPara.coalStatic.tx_comp_depth =
1288 			(ec->tx_max_coalesced_frames ?
1289 			 ec->tx_max_coalesced_frames :
1290 			 VMXNET3_COAL_STATIC_DEFAULT_DEPTH);
1291 
1292 		adapter->coal_conf->coalPara.coalStatic.rx_depth =
1293 			(ec->rx_max_coalesced_frames ?
1294 			 ec->rx_max_coalesced_frames :
1295 			 VMXNET3_COAL_STATIC_DEFAULT_DEPTH);
1296 
1297 		adapter->coal_conf->coalPara.coalStatic.tx_depth =
1298 			 VMXNET3_COAL_STATIC_DEFAULT_DEPTH;
1299 		goto done;
1300 	}
1301 
1302 done:
1303 	adapter->default_coal_mode = false;
1304 	if (netif_running(netdev)) {
1305 		spin_lock_irqsave(&adapter->cmd_lock, flags);
1306 		cmdInfo->varConf.confVer = 1;
1307 		cmdInfo->varConf.confLen =
1308 			cpu_to_le32(sizeof(*adapter->coal_conf));
1309 		cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
1310 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
1311 				       VMXNET3_CMD_SET_COALESCE);
1312 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
1313 	}
1314 
1315 	return 0;
1316 }
1317 
1318 static void vmxnet3_get_channels(struct net_device *netdev,
1319 				 struct ethtool_channels *ec)
1320 {
1321 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1322 
1323 	if (IS_ENABLED(CONFIG_PCI_MSI) && adapter->intr.type == VMXNET3_IT_MSIX) {
1324 		if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
1325 			ec->combined_count = adapter->num_tx_queues;
1326 		} else {
1327 			ec->rx_count = adapter->num_rx_queues;
1328 			ec->tx_count =
1329 				adapter->share_intr == VMXNET3_INTR_TXSHARE ?
1330 					       1 : adapter->num_tx_queues;
1331 		}
1332 	} else {
1333 		ec->combined_count = 1;
1334 	}
1335 
1336 	ec->other_count = 1;
1337 
1338 	/* Number of interrupts cannot be changed on the fly */
1339 	/* Just set maximums to actual values */
1340 	ec->max_rx = ec->rx_count;
1341 	ec->max_tx = ec->tx_count;
1342 	ec->max_combined = ec->combined_count;
1343 	ec->max_other = ec->other_count;
1344 }
1345 
1346 static const struct ethtool_ops vmxnet3_ethtool_ops = {
1347 	.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
1348 				     ETHTOOL_COALESCE_MAX_FRAMES |
1349 				     ETHTOOL_COALESCE_USE_ADAPTIVE_RX,
1350 	.get_drvinfo       = vmxnet3_get_drvinfo,
1351 	.get_regs_len      = vmxnet3_get_regs_len,
1352 	.get_regs          = vmxnet3_get_regs,
1353 	.get_wol           = vmxnet3_get_wol,
1354 	.set_wol           = vmxnet3_set_wol,
1355 	.get_link          = ethtool_op_get_link,
1356 	.get_coalesce      = vmxnet3_get_coalesce,
1357 	.set_coalesce      = vmxnet3_set_coalesce,
1358 	.get_strings       = vmxnet3_get_strings,
1359 	.get_sset_count	   = vmxnet3_get_sset_count,
1360 	.get_ethtool_stats = vmxnet3_get_ethtool_stats,
1361 	.get_ringparam     = vmxnet3_get_ringparam,
1362 	.set_ringparam     = vmxnet3_set_ringparam,
1363 	.get_rxnfc         = vmxnet3_get_rxnfc,
1364 	.set_rxnfc         = vmxnet3_set_rxnfc,
1365 #ifdef VMXNET3_RSS
1366 	.get_rxfh_indir_size = vmxnet3_get_rss_indir_size,
1367 	.get_rxfh          = vmxnet3_get_rss,
1368 	.set_rxfh          = vmxnet3_set_rss,
1369 #endif
1370 	.get_link_ksettings = vmxnet3_get_link_ksettings,
1371 	.get_channels      = vmxnet3_get_channels,
1372 };
1373 
1374 void vmxnet3_set_ethtool_ops(struct net_device *netdev)
1375 {
1376 	netdev->ethtool_ops = &vmxnet3_ethtool_ops;
1377 }
1378