1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2019 Google, Inc.
5  */
6 
7 #include <linux/ethtool.h>
8 #include <linux/rtnetlink.h>
9 #include "gve.h"
10 #include "gve_adminq.h"
11 
12 static void gve_get_drvinfo(struct net_device *netdev,
13 			    struct ethtool_drvinfo *info)
14 {
15 	struct gve_priv *priv = netdev_priv(netdev);
16 
17 	strlcpy(info->driver, "gve", sizeof(info->driver));
18 	strlcpy(info->version, gve_version_str, sizeof(info->version));
19 	strlcpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info));
20 }
21 
22 static void gve_set_msglevel(struct net_device *netdev, u32 value)
23 {
24 	struct gve_priv *priv = netdev_priv(netdev);
25 
26 	priv->msg_enable = value;
27 }
28 
29 static u32 gve_get_msglevel(struct net_device *netdev)
30 {
31 	struct gve_priv *priv = netdev_priv(netdev);
32 
33 	return priv->msg_enable;
34 }
35 
36 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {
37 	"rx_packets", "tx_packets", "rx_bytes", "tx_bytes",
38 	"rx_dropped", "tx_dropped", "tx_timeouts",
39 	"rx_skb_alloc_fail", "rx_buf_alloc_fail", "rx_desc_err_dropped_pkt",
40 	"interface_up_cnt", "interface_down_cnt", "reset_cnt",
41 	"page_alloc_fail", "dma_mapping_error", "stats_report_trigger_cnt",
42 };
43 
44 static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = {
45 	"rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_bytes[%u]",
46 	"rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]",
47 	"rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]",
48 	"rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]",
49 };
50 
51 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = {
52 	"tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_bytes[%u]",
53 	"tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]",
54 	"tx_dma_mapping_error[%u]",
55 };
56 
57 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] = {
58 	"adminq_prod_cnt", "adminq_cmd_fail", "adminq_timeouts",
59 	"adminq_describe_device_cnt", "adminq_cfg_device_resources_cnt",
60 	"adminq_register_page_list_cnt", "adminq_unregister_page_list_cnt",
61 	"adminq_create_tx_queue_cnt", "adminq_create_rx_queue_cnt",
62 	"adminq_destroy_tx_queue_cnt", "adminq_destroy_rx_queue_cnt",
63 	"adminq_dcfg_device_resources_cnt", "adminq_set_driver_parameter_cnt",
64 	"adminq_report_stats_cnt", "adminq_report_link_speed_cnt"
65 };
66 
67 static const char gve_gstrings_priv_flags[][ETH_GSTRING_LEN] = {
68 	"report-stats",
69 };
70 
71 #define GVE_MAIN_STATS_LEN  ARRAY_SIZE(gve_gstrings_main_stats)
72 #define GVE_ADMINQ_STATS_LEN  ARRAY_SIZE(gve_gstrings_adminq_stats)
73 #define NUM_GVE_TX_CNTS	ARRAY_SIZE(gve_gstrings_tx_stats)
74 #define NUM_GVE_RX_CNTS	ARRAY_SIZE(gve_gstrings_rx_stats)
75 #define GVE_PRIV_FLAGS_STR_LEN ARRAY_SIZE(gve_gstrings_priv_flags)
76 
77 static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
78 {
79 	struct gve_priv *priv = netdev_priv(netdev);
80 	char *s = (char *)data;
81 	int i, j;
82 
83 	switch (stringset) {
84 	case ETH_SS_STATS:
85 		memcpy(s, *gve_gstrings_main_stats,
86 		       sizeof(gve_gstrings_main_stats));
87 		s += sizeof(gve_gstrings_main_stats);
88 
89 		for (i = 0; i < priv->rx_cfg.num_queues; i++) {
90 			for (j = 0; j < NUM_GVE_RX_CNTS; j++) {
91 				snprintf(s, ETH_GSTRING_LEN,
92 					 gve_gstrings_rx_stats[j], i);
93 				s += ETH_GSTRING_LEN;
94 			}
95 		}
96 
97 		for (i = 0; i < priv->tx_cfg.num_queues; i++) {
98 			for (j = 0; j < NUM_GVE_TX_CNTS; j++) {
99 				snprintf(s, ETH_GSTRING_LEN,
100 					 gve_gstrings_tx_stats[j], i);
101 				s += ETH_GSTRING_LEN;
102 			}
103 		}
104 
105 		memcpy(s, *gve_gstrings_adminq_stats,
106 		       sizeof(gve_gstrings_adminq_stats));
107 		s += sizeof(gve_gstrings_adminq_stats);
108 		break;
109 
110 	case ETH_SS_PRIV_FLAGS:
111 		memcpy(s, *gve_gstrings_priv_flags,
112 		       sizeof(gve_gstrings_priv_flags));
113 		s += sizeof(gve_gstrings_priv_flags);
114 		break;
115 
116 	default:
117 		break;
118 	}
119 }
120 
121 static int gve_get_sset_count(struct net_device *netdev, int sset)
122 {
123 	struct gve_priv *priv = netdev_priv(netdev);
124 
125 	switch (sset) {
126 	case ETH_SS_STATS:
127 		return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN +
128 		       (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) +
129 		       (priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS);
130 	case ETH_SS_PRIV_FLAGS:
131 		return GVE_PRIV_FLAGS_STR_LEN;
132 	default:
133 		return -EOPNOTSUPP;
134 	}
135 }
136 
137 static void
138 gve_get_ethtool_stats(struct net_device *netdev,
139 		      struct ethtool_stats *stats, u64 *data)
140 {
141 	u64 tmp_rx_pkts, tmp_rx_bytes, tmp_rx_skb_alloc_fail,	tmp_rx_buf_alloc_fail,
142 		tmp_rx_desc_err_dropped_pkt, tmp_tx_pkts, tmp_tx_bytes;
143 	u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_pkts,
144 		rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes;
145 	int stats_idx, base_stats_idx, max_stats_idx;
146 	struct stats *report_stats;
147 	int *rx_qid_to_stats_idx;
148 	int *tx_qid_to_stats_idx;
149 	struct gve_priv *priv;
150 	bool skip_nic_stats;
151 	unsigned int start;
152 	int ring;
153 	int i, j;
154 
155 	ASSERT_RTNL();
156 
157 	priv = netdev_priv(netdev);
158 	report_stats = priv->stats_report->stats;
159 	rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues,
160 					    sizeof(int), GFP_KERNEL);
161 	if (!rx_qid_to_stats_idx)
162 		return;
163 	tx_qid_to_stats_idx = kmalloc_array(priv->tx_cfg.num_queues,
164 					    sizeof(int), GFP_KERNEL);
165 	if (!tx_qid_to_stats_idx) {
166 		kfree(rx_qid_to_stats_idx);
167 		return;
168 	}
169 	for (rx_pkts = 0, rx_bytes = 0, rx_skb_alloc_fail = 0,
170 	     rx_buf_alloc_fail = 0, rx_desc_err_dropped_pkt = 0, ring = 0;
171 	     ring < priv->rx_cfg.num_queues; ring++) {
172 		if (priv->rx) {
173 			do {
174 				struct gve_rx_ring *rx = &priv->rx[ring];
175 
176 				start =
177 				  u64_stats_fetch_begin(&priv->rx[ring].statss);
178 				tmp_rx_pkts = rx->rpackets;
179 				tmp_rx_bytes = rx->rbytes;
180 				tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail;
181 				tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail;
182 				tmp_rx_desc_err_dropped_pkt =
183 					rx->rx_desc_err_dropped_pkt;
184 			} while (u64_stats_fetch_retry(&priv->rx[ring].statss,
185 						       start));
186 			rx_pkts += tmp_rx_pkts;
187 			rx_bytes += tmp_rx_bytes;
188 			rx_skb_alloc_fail += tmp_rx_skb_alloc_fail;
189 			rx_buf_alloc_fail += tmp_rx_buf_alloc_fail;
190 			rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt;
191 		}
192 	}
193 	for (tx_pkts = 0, tx_bytes = 0, ring = 0;
194 	     ring < priv->tx_cfg.num_queues; ring++) {
195 		if (priv->tx) {
196 			do {
197 				start =
198 				  u64_stats_fetch_begin(&priv->tx[ring].statss);
199 				tmp_tx_pkts = priv->tx[ring].pkt_done;
200 				tmp_tx_bytes = priv->tx[ring].bytes_done;
201 			} while (u64_stats_fetch_retry(&priv->tx[ring].statss,
202 						       start));
203 			tx_pkts += tmp_tx_pkts;
204 			tx_bytes += tmp_tx_bytes;
205 		}
206 	}
207 
208 	i = 0;
209 	data[i++] = rx_pkts;
210 	data[i++] = tx_pkts;
211 	data[i++] = rx_bytes;
212 	data[i++] = tx_bytes;
213 	/* total rx dropped packets */
214 	data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail +
215 		    rx_desc_err_dropped_pkt;
216 	/* Skip tx_dropped */
217 	i++;
218 
219 	data[i++] = priv->tx_timeo_cnt;
220 	data[i++] = rx_skb_alloc_fail;
221 	data[i++] = rx_buf_alloc_fail;
222 	data[i++] = rx_desc_err_dropped_pkt;
223 	data[i++] = priv->interface_up_cnt;
224 	data[i++] = priv->interface_down_cnt;
225 	data[i++] = priv->reset_cnt;
226 	data[i++] = priv->page_alloc_fail;
227 	data[i++] = priv->dma_mapping_error;
228 	data[i++] = priv->stats_report_trigger_cnt;
229 	i = GVE_MAIN_STATS_LEN;
230 
231 	/* For rx cross-reporting stats, start from nic rx stats in report */
232 	base_stats_idx = GVE_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues +
233 		GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues;
234 	max_stats_idx = NIC_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues +
235 		base_stats_idx;
236 	/* Preprocess the stats report for rx, map queue id to start index */
237 	skip_nic_stats = false;
238 	for (stats_idx = base_stats_idx; stats_idx < max_stats_idx;
239 		stats_idx += NIC_RX_STATS_REPORT_NUM) {
240 		u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name);
241 		u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id);
242 
243 		if (stat_name == 0) {
244 			/* no stats written by NIC yet */
245 			skip_nic_stats = true;
246 			break;
247 		}
248 		rx_qid_to_stats_idx[queue_id] = stats_idx;
249 	}
250 	/* walk RX rings */
251 	if (priv->rx) {
252 		for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
253 			struct gve_rx_ring *rx = &priv->rx[ring];
254 
255 			data[i++] = rx->fill_cnt;
256 			data[i++] = rx->cnt;
257 			do {
258 				start =
259 				  u64_stats_fetch_begin(&priv->rx[ring].statss);
260 				tmp_rx_bytes = rx->rbytes;
261 				tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail;
262 				tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail;
263 				tmp_rx_desc_err_dropped_pkt =
264 					rx->rx_desc_err_dropped_pkt;
265 			} while (u64_stats_fetch_retry(&priv->rx[ring].statss,
266 						       start));
267 			data[i++] = tmp_rx_bytes;
268 			/* rx dropped packets */
269 			data[i++] = tmp_rx_skb_alloc_fail +
270 				tmp_rx_buf_alloc_fail +
271 				tmp_rx_desc_err_dropped_pkt;
272 			data[i++] = rx->rx_copybreak_pkt;
273 			data[i++] = rx->rx_copied_pkt;
274 			/* stats from NIC */
275 			if (skip_nic_stats) {
276 				/* skip NIC rx stats */
277 				i += NIC_RX_STATS_REPORT_NUM;
278 				continue;
279 			}
280 			for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) {
281 				u64 value =
282 				be64_to_cpu(report_stats[rx_qid_to_stats_idx[ring] + j].value);
283 
284 				data[i++] = value;
285 			}
286 		}
287 	} else {
288 		i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS;
289 	}
290 
291 	/* For tx cross-reporting stats, start from nic tx stats in report */
292 	base_stats_idx = max_stats_idx;
293 	max_stats_idx = NIC_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues +
294 		max_stats_idx;
295 	/* Preprocess the stats report for tx, map queue id to start index */
296 	skip_nic_stats = false;
297 	for (stats_idx = base_stats_idx; stats_idx < max_stats_idx;
298 		stats_idx += NIC_TX_STATS_REPORT_NUM) {
299 		u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name);
300 		u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id);
301 
302 		if (stat_name == 0) {
303 			/* no stats written by NIC yet */
304 			skip_nic_stats = true;
305 			break;
306 		}
307 		tx_qid_to_stats_idx[queue_id] = stats_idx;
308 	}
309 	/* walk TX rings */
310 	if (priv->tx) {
311 		for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) {
312 			struct gve_tx_ring *tx = &priv->tx[ring];
313 
314 			data[i++] = tx->req;
315 			data[i++] = tx->done;
316 			do {
317 				start =
318 				  u64_stats_fetch_begin(&priv->tx[ring].statss);
319 				tmp_tx_bytes = tx->bytes_done;
320 			} while (u64_stats_fetch_retry(&priv->tx[ring].statss,
321 						       start));
322 			data[i++] = tmp_tx_bytes;
323 			data[i++] = tx->wake_queue;
324 			data[i++] = tx->stop_queue;
325 			data[i++] = be32_to_cpu(gve_tx_load_event_counter(priv,
326 									  tx));
327 			data[i++] = tx->dma_mapping_error;
328 			/* stats from NIC */
329 			if (skip_nic_stats) {
330 				/* skip NIC tx stats */
331 				i += NIC_TX_STATS_REPORT_NUM;
332 				continue;
333 			}
334 			for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) {
335 				u64 value =
336 				be64_to_cpu(report_stats[tx_qid_to_stats_idx[ring] + j].value);
337 				data[i++] = value;
338 			}
339 		}
340 	} else {
341 		i += priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS;
342 	}
343 
344 	kfree(rx_qid_to_stats_idx);
345 	kfree(tx_qid_to_stats_idx);
346 	/* AQ Stats */
347 	data[i++] = priv->adminq_prod_cnt;
348 	data[i++] = priv->adminq_cmd_fail;
349 	data[i++] = priv->adminq_timeouts;
350 	data[i++] = priv->adminq_describe_device_cnt;
351 	data[i++] = priv->adminq_cfg_device_resources_cnt;
352 	data[i++] = priv->adminq_register_page_list_cnt;
353 	data[i++] = priv->adminq_unregister_page_list_cnt;
354 	data[i++] = priv->adminq_create_tx_queue_cnt;
355 	data[i++] = priv->adminq_create_rx_queue_cnt;
356 	data[i++] = priv->adminq_destroy_tx_queue_cnt;
357 	data[i++] = priv->adminq_destroy_rx_queue_cnt;
358 	data[i++] = priv->adminq_dcfg_device_resources_cnt;
359 	data[i++] = priv->adminq_set_driver_parameter_cnt;
360 	data[i++] = priv->adminq_report_stats_cnt;
361 	data[i++] = priv->adminq_report_link_speed_cnt;
362 }
363 
364 static void gve_get_channels(struct net_device *netdev,
365 			     struct ethtool_channels *cmd)
366 {
367 	struct gve_priv *priv = netdev_priv(netdev);
368 
369 	cmd->max_rx = priv->rx_cfg.max_queues;
370 	cmd->max_tx = priv->tx_cfg.max_queues;
371 	cmd->max_other = 0;
372 	cmd->max_combined = 0;
373 	cmd->rx_count = priv->rx_cfg.num_queues;
374 	cmd->tx_count = priv->tx_cfg.num_queues;
375 	cmd->other_count = 0;
376 	cmd->combined_count = 0;
377 }
378 
379 static int gve_set_channels(struct net_device *netdev,
380 			    struct ethtool_channels *cmd)
381 {
382 	struct gve_priv *priv = netdev_priv(netdev);
383 	struct gve_queue_config new_tx_cfg = priv->tx_cfg;
384 	struct gve_queue_config new_rx_cfg = priv->rx_cfg;
385 	struct ethtool_channels old_settings;
386 	int new_tx = cmd->tx_count;
387 	int new_rx = cmd->rx_count;
388 
389 	gve_get_channels(netdev, &old_settings);
390 
391 	/* Changing combined is not allowed allowed */
392 	if (cmd->combined_count != old_settings.combined_count)
393 		return -EINVAL;
394 
395 	if (!new_rx || !new_tx)
396 		return -EINVAL;
397 
398 	if (!netif_carrier_ok(netdev)) {
399 		priv->tx_cfg.num_queues = new_tx;
400 		priv->rx_cfg.num_queues = new_rx;
401 		return 0;
402 	}
403 
404 	new_tx_cfg.num_queues = new_tx;
405 	new_rx_cfg.num_queues = new_rx;
406 
407 	return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg);
408 }
409 
410 static void gve_get_ringparam(struct net_device *netdev,
411 			      struct ethtool_ringparam *cmd)
412 {
413 	struct gve_priv *priv = netdev_priv(netdev);
414 
415 	cmd->rx_max_pending = priv->rx_desc_cnt;
416 	cmd->tx_max_pending = priv->tx_desc_cnt;
417 	cmd->rx_pending = priv->rx_desc_cnt;
418 	cmd->tx_pending = priv->tx_desc_cnt;
419 }
420 
421 static int gve_user_reset(struct net_device *netdev, u32 *flags)
422 {
423 	struct gve_priv *priv = netdev_priv(netdev);
424 
425 	if (*flags == ETH_RESET_ALL) {
426 		*flags = 0;
427 		return gve_reset(priv, true);
428 	}
429 
430 	return -EOPNOTSUPP;
431 }
432 
433 static int gve_get_tunable(struct net_device *netdev,
434 			   const struct ethtool_tunable *etuna, void *value)
435 {
436 	struct gve_priv *priv = netdev_priv(netdev);
437 
438 	switch (etuna->id) {
439 	case ETHTOOL_RX_COPYBREAK:
440 		*(u32 *)value = priv->rx_copybreak;
441 		return 0;
442 	default:
443 		return -EOPNOTSUPP;
444 	}
445 }
446 
447 static int gve_set_tunable(struct net_device *netdev,
448 			   const struct ethtool_tunable *etuna,
449 			   const void *value)
450 {
451 	struct gve_priv *priv = netdev_priv(netdev);
452 	u32 len;
453 
454 	switch (etuna->id) {
455 	case ETHTOOL_RX_COPYBREAK:
456 		len = *(u32 *)value;
457 		if (len > PAGE_SIZE / 2)
458 			return -EINVAL;
459 		priv->rx_copybreak = len;
460 		return 0;
461 	default:
462 		return -EOPNOTSUPP;
463 	}
464 }
465 
466 static u32 gve_get_priv_flags(struct net_device *netdev)
467 {
468 	struct gve_priv *priv = netdev_priv(netdev);
469 	u32 ret_flags = 0;
470 
471 	/* Only 1 flag exists currently: report-stats (BIT(O)), so set that flag. */
472 	if (priv->ethtool_flags & BIT(0))
473 		ret_flags |= BIT(0);
474 	return ret_flags;
475 }
476 
477 static int gve_set_priv_flags(struct net_device *netdev, u32 flags)
478 {
479 	struct gve_priv *priv = netdev_priv(netdev);
480 	u64 ori_flags, new_flags;
481 
482 	ori_flags = READ_ONCE(priv->ethtool_flags);
483 	new_flags = ori_flags;
484 
485 	/* Only one priv flag exists: report-stats (BIT(0))*/
486 	if (flags & BIT(0))
487 		new_flags |= BIT(0);
488 	else
489 		new_flags &= ~(BIT(0));
490 	priv->ethtool_flags = new_flags;
491 	/* start report-stats timer when user turns report stats on. */
492 	if (flags & BIT(0)) {
493 		mod_timer(&priv->stats_report_timer,
494 			  round_jiffies(jiffies +
495 					msecs_to_jiffies(priv->stats_report_timer_period)));
496 	}
497 	/* Zero off gve stats when report-stats turned off and */
498 	/* delete report stats timer. */
499 	if (!(flags & BIT(0)) && (ori_flags & BIT(0))) {
500 		int tx_stats_num = GVE_TX_STATS_REPORT_NUM *
501 			priv->tx_cfg.num_queues;
502 		int rx_stats_num = GVE_RX_STATS_REPORT_NUM *
503 			priv->rx_cfg.num_queues;
504 
505 		memset(priv->stats_report->stats, 0, (tx_stats_num + rx_stats_num) *
506 				   sizeof(struct stats));
507 		del_timer_sync(&priv->stats_report_timer);
508 	}
509 	return 0;
510 }
511 
512 static int gve_get_link_ksettings(struct net_device *netdev,
513 				  struct ethtool_link_ksettings *cmd)
514 {
515 	struct gve_priv *priv = netdev_priv(netdev);
516 	int err = gve_adminq_report_link_speed(priv);
517 
518 	cmd->base.speed = priv->link_speed;
519 	return err;
520 }
521 
522 const struct ethtool_ops gve_ethtool_ops = {
523 	.get_drvinfo = gve_get_drvinfo,
524 	.get_strings = gve_get_strings,
525 	.get_sset_count = gve_get_sset_count,
526 	.get_ethtool_stats = gve_get_ethtool_stats,
527 	.set_msglevel = gve_set_msglevel,
528 	.get_msglevel = gve_get_msglevel,
529 	.set_channels = gve_set_channels,
530 	.get_channels = gve_get_channels,
531 	.get_link = ethtool_op_get_link,
532 	.get_ringparam = gve_get_ringparam,
533 	.reset = gve_user_reset,
534 	.get_tunable = gve_get_tunable,
535 	.set_tunable = gve_set_tunable,
536 	.get_priv_flags = gve_get_priv_flags,
537 	.set_priv_flags = gve_set_priv_flags,
538 	.get_link_ksettings = gve_get_link_ksettings
539 };
540