1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/ethtool.h> 8 #include <linux/rtnetlink.h> 9 #include "gve.h" 10 #include "gve_adminq.h" 11 12 static void gve_get_drvinfo(struct net_device *netdev, 13 struct ethtool_drvinfo *info) 14 { 15 struct gve_priv *priv = netdev_priv(netdev); 16 17 strscpy(info->driver, "gve", sizeof(info->driver)); 18 strscpy(info->version, gve_version_str, sizeof(info->version)); 19 strscpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info)); 20 } 21 22 static void gve_set_msglevel(struct net_device *netdev, u32 value) 23 { 24 struct gve_priv *priv = netdev_priv(netdev); 25 26 priv->msg_enable = value; 27 } 28 29 static u32 gve_get_msglevel(struct net_device *netdev) 30 { 31 struct gve_priv *priv = netdev_priv(netdev); 32 33 return priv->msg_enable; 34 } 35 36 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = { 37 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", 38 "rx_dropped", "tx_dropped", "tx_timeouts", 39 "rx_skb_alloc_fail", "rx_buf_alloc_fail", "rx_desc_err_dropped_pkt", 40 "interface_up_cnt", "interface_down_cnt", "reset_cnt", 41 "page_alloc_fail", "dma_mapping_error", "stats_report_trigger_cnt", 42 }; 43 44 static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = { 45 "rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_bytes[%u]", 46 "rx_cont_packet_cnt[%u]", "rx_frag_flip_cnt[%u]", "rx_frag_copy_cnt[%u]", 47 "rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]", 48 "rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]", 49 "rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]", 50 }; 51 52 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = { 53 "tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_bytes[%u]", 54 "tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]", 55 "tx_dma_mapping_error[%u]", 56 }; 57 58 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] = { 59 "adminq_prod_cnt", "adminq_cmd_fail", "adminq_timeouts", 60 "adminq_describe_device_cnt", "adminq_cfg_device_resources_cnt", 61 "adminq_register_page_list_cnt", "adminq_unregister_page_list_cnt", 62 "adminq_create_tx_queue_cnt", "adminq_create_rx_queue_cnt", 63 "adminq_destroy_tx_queue_cnt", "adminq_destroy_rx_queue_cnt", 64 "adminq_dcfg_device_resources_cnt", "adminq_set_driver_parameter_cnt", 65 "adminq_report_stats_cnt", "adminq_report_link_speed_cnt" 66 }; 67 68 static const char gve_gstrings_priv_flags[][ETH_GSTRING_LEN] = { 69 "report-stats", 70 }; 71 72 #define GVE_MAIN_STATS_LEN ARRAY_SIZE(gve_gstrings_main_stats) 73 #define GVE_ADMINQ_STATS_LEN ARRAY_SIZE(gve_gstrings_adminq_stats) 74 #define NUM_GVE_TX_CNTS ARRAY_SIZE(gve_gstrings_tx_stats) 75 #define NUM_GVE_RX_CNTS ARRAY_SIZE(gve_gstrings_rx_stats) 76 #define GVE_PRIV_FLAGS_STR_LEN ARRAY_SIZE(gve_gstrings_priv_flags) 77 78 static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data) 79 { 80 struct gve_priv *priv = netdev_priv(netdev); 81 char *s = (char *)data; 82 int i, j; 83 84 switch (stringset) { 85 case ETH_SS_STATS: 86 memcpy(s, *gve_gstrings_main_stats, 87 sizeof(gve_gstrings_main_stats)); 88 s += sizeof(gve_gstrings_main_stats); 89 90 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 91 for (j = 0; j < NUM_GVE_RX_CNTS; j++) { 92 snprintf(s, ETH_GSTRING_LEN, 93 gve_gstrings_rx_stats[j], i); 94 s += ETH_GSTRING_LEN; 95 } 96 } 97 98 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 99 for (j = 0; j < NUM_GVE_TX_CNTS; j++) { 100 snprintf(s, ETH_GSTRING_LEN, 101 gve_gstrings_tx_stats[j], i); 102 s += ETH_GSTRING_LEN; 103 } 104 } 105 106 memcpy(s, *gve_gstrings_adminq_stats, 107 sizeof(gve_gstrings_adminq_stats)); 108 s += sizeof(gve_gstrings_adminq_stats); 109 break; 110 111 case ETH_SS_PRIV_FLAGS: 112 memcpy(s, *gve_gstrings_priv_flags, 113 sizeof(gve_gstrings_priv_flags)); 114 s += sizeof(gve_gstrings_priv_flags); 115 break; 116 117 default: 118 break; 119 } 120 } 121 122 static int gve_get_sset_count(struct net_device *netdev, int sset) 123 { 124 struct gve_priv *priv = netdev_priv(netdev); 125 126 switch (sset) { 127 case ETH_SS_STATS: 128 return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN + 129 (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) + 130 (priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS); 131 case ETH_SS_PRIV_FLAGS: 132 return GVE_PRIV_FLAGS_STR_LEN; 133 default: 134 return -EOPNOTSUPP; 135 } 136 } 137 138 static void 139 gve_get_ethtool_stats(struct net_device *netdev, 140 struct ethtool_stats *stats, u64 *data) 141 { 142 u64 tmp_rx_pkts, tmp_rx_bytes, tmp_rx_skb_alloc_fail, tmp_rx_buf_alloc_fail, 143 tmp_rx_desc_err_dropped_pkt, tmp_tx_pkts, tmp_tx_bytes; 144 u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_pkts, 145 rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes; 146 int stats_idx, base_stats_idx, max_stats_idx; 147 struct stats *report_stats; 148 int *rx_qid_to_stats_idx; 149 int *tx_qid_to_stats_idx; 150 struct gve_priv *priv; 151 bool skip_nic_stats; 152 unsigned int start; 153 int ring; 154 int i, j; 155 156 ASSERT_RTNL(); 157 158 priv = netdev_priv(netdev); 159 report_stats = priv->stats_report->stats; 160 rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues, 161 sizeof(int), GFP_KERNEL); 162 if (!rx_qid_to_stats_idx) 163 return; 164 tx_qid_to_stats_idx = kmalloc_array(priv->tx_cfg.num_queues, 165 sizeof(int), GFP_KERNEL); 166 if (!tx_qid_to_stats_idx) { 167 kfree(rx_qid_to_stats_idx); 168 return; 169 } 170 for (rx_pkts = 0, rx_bytes = 0, rx_skb_alloc_fail = 0, 171 rx_buf_alloc_fail = 0, rx_desc_err_dropped_pkt = 0, ring = 0; 172 ring < priv->rx_cfg.num_queues; ring++) { 173 if (priv->rx) { 174 do { 175 struct gve_rx_ring *rx = &priv->rx[ring]; 176 177 start = 178 u64_stats_fetch_begin(&priv->rx[ring].statss); 179 tmp_rx_pkts = rx->rpackets; 180 tmp_rx_bytes = rx->rbytes; 181 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 182 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 183 tmp_rx_desc_err_dropped_pkt = 184 rx->rx_desc_err_dropped_pkt; 185 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 186 start)); 187 rx_pkts += tmp_rx_pkts; 188 rx_bytes += tmp_rx_bytes; 189 rx_skb_alloc_fail += tmp_rx_skb_alloc_fail; 190 rx_buf_alloc_fail += tmp_rx_buf_alloc_fail; 191 rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt; 192 } 193 } 194 for (tx_pkts = 0, tx_bytes = 0, ring = 0; 195 ring < priv->tx_cfg.num_queues; ring++) { 196 if (priv->tx) { 197 do { 198 start = 199 u64_stats_fetch_begin(&priv->tx[ring].statss); 200 tmp_tx_pkts = priv->tx[ring].pkt_done; 201 tmp_tx_bytes = priv->tx[ring].bytes_done; 202 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 203 start)); 204 tx_pkts += tmp_tx_pkts; 205 tx_bytes += tmp_tx_bytes; 206 } 207 } 208 209 i = 0; 210 data[i++] = rx_pkts; 211 data[i++] = tx_pkts; 212 data[i++] = rx_bytes; 213 data[i++] = tx_bytes; 214 /* total rx dropped packets */ 215 data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail + 216 rx_desc_err_dropped_pkt; 217 /* Skip tx_dropped */ 218 i++; 219 220 data[i++] = priv->tx_timeo_cnt; 221 data[i++] = rx_skb_alloc_fail; 222 data[i++] = rx_buf_alloc_fail; 223 data[i++] = rx_desc_err_dropped_pkt; 224 data[i++] = priv->interface_up_cnt; 225 data[i++] = priv->interface_down_cnt; 226 data[i++] = priv->reset_cnt; 227 data[i++] = priv->page_alloc_fail; 228 data[i++] = priv->dma_mapping_error; 229 data[i++] = priv->stats_report_trigger_cnt; 230 i = GVE_MAIN_STATS_LEN; 231 232 /* For rx cross-reporting stats, start from nic rx stats in report */ 233 base_stats_idx = GVE_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues + 234 GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues; 235 max_stats_idx = NIC_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues + 236 base_stats_idx; 237 /* Preprocess the stats report for rx, map queue id to start index */ 238 skip_nic_stats = false; 239 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 240 stats_idx += NIC_RX_STATS_REPORT_NUM) { 241 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 242 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 243 244 if (stat_name == 0) { 245 /* no stats written by NIC yet */ 246 skip_nic_stats = true; 247 break; 248 } 249 rx_qid_to_stats_idx[queue_id] = stats_idx; 250 } 251 /* walk RX rings */ 252 if (priv->rx) { 253 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 254 struct gve_rx_ring *rx = &priv->rx[ring]; 255 256 data[i++] = rx->fill_cnt; 257 data[i++] = rx->cnt; 258 do { 259 start = 260 u64_stats_fetch_begin(&priv->rx[ring].statss); 261 tmp_rx_bytes = rx->rbytes; 262 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 263 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 264 tmp_rx_desc_err_dropped_pkt = 265 rx->rx_desc_err_dropped_pkt; 266 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 267 start)); 268 data[i++] = tmp_rx_bytes; 269 data[i++] = rx->rx_cont_packet_cnt; 270 data[i++] = rx->rx_frag_flip_cnt; 271 data[i++] = rx->rx_frag_copy_cnt; 272 /* rx dropped packets */ 273 data[i++] = tmp_rx_skb_alloc_fail + 274 tmp_rx_buf_alloc_fail + 275 tmp_rx_desc_err_dropped_pkt; 276 data[i++] = rx->rx_copybreak_pkt; 277 data[i++] = rx->rx_copied_pkt; 278 /* stats from NIC */ 279 if (skip_nic_stats) { 280 /* skip NIC rx stats */ 281 i += NIC_RX_STATS_REPORT_NUM; 282 continue; 283 } 284 for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) { 285 u64 value = 286 be64_to_cpu(report_stats[rx_qid_to_stats_idx[ring] + j].value); 287 288 data[i++] = value; 289 } 290 } 291 } else { 292 i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS; 293 } 294 295 /* For tx cross-reporting stats, start from nic tx stats in report */ 296 base_stats_idx = max_stats_idx; 297 max_stats_idx = NIC_TX_STATS_REPORT_NUM * priv->tx_cfg.num_queues + 298 max_stats_idx; 299 /* Preprocess the stats report for tx, map queue id to start index */ 300 skip_nic_stats = false; 301 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 302 stats_idx += NIC_TX_STATS_REPORT_NUM) { 303 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 304 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 305 306 if (stat_name == 0) { 307 /* no stats written by NIC yet */ 308 skip_nic_stats = true; 309 break; 310 } 311 tx_qid_to_stats_idx[queue_id] = stats_idx; 312 } 313 /* walk TX rings */ 314 if (priv->tx) { 315 for (ring = 0; ring < priv->tx_cfg.num_queues; ring++) { 316 struct gve_tx_ring *tx = &priv->tx[ring]; 317 318 if (gve_is_gqi(priv)) { 319 data[i++] = tx->req; 320 data[i++] = tx->done; 321 } else { 322 /* DQO doesn't currently support 323 * posted/completed descriptor counts; 324 */ 325 data[i++] = 0; 326 data[i++] = 0; 327 } 328 do { 329 start = 330 u64_stats_fetch_begin(&priv->tx[ring].statss); 331 tmp_tx_bytes = tx->bytes_done; 332 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 333 start)); 334 data[i++] = tmp_tx_bytes; 335 data[i++] = tx->wake_queue; 336 data[i++] = tx->stop_queue; 337 data[i++] = gve_tx_load_event_counter(priv, tx); 338 data[i++] = tx->dma_mapping_error; 339 /* stats from NIC */ 340 if (skip_nic_stats) { 341 /* skip NIC tx stats */ 342 i += NIC_TX_STATS_REPORT_NUM; 343 continue; 344 } 345 for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) { 346 u64 value = 347 be64_to_cpu(report_stats[tx_qid_to_stats_idx[ring] + j].value); 348 data[i++] = value; 349 } 350 } 351 } else { 352 i += priv->tx_cfg.num_queues * NUM_GVE_TX_CNTS; 353 } 354 355 kfree(rx_qid_to_stats_idx); 356 kfree(tx_qid_to_stats_idx); 357 /* AQ Stats */ 358 data[i++] = priv->adminq_prod_cnt; 359 data[i++] = priv->adminq_cmd_fail; 360 data[i++] = priv->adminq_timeouts; 361 data[i++] = priv->adminq_describe_device_cnt; 362 data[i++] = priv->adminq_cfg_device_resources_cnt; 363 data[i++] = priv->adminq_register_page_list_cnt; 364 data[i++] = priv->adminq_unregister_page_list_cnt; 365 data[i++] = priv->adminq_create_tx_queue_cnt; 366 data[i++] = priv->adminq_create_rx_queue_cnt; 367 data[i++] = priv->adminq_destroy_tx_queue_cnt; 368 data[i++] = priv->adminq_destroy_rx_queue_cnt; 369 data[i++] = priv->adminq_dcfg_device_resources_cnt; 370 data[i++] = priv->adminq_set_driver_parameter_cnt; 371 data[i++] = priv->adminq_report_stats_cnt; 372 data[i++] = priv->adminq_report_link_speed_cnt; 373 } 374 375 static void gve_get_channels(struct net_device *netdev, 376 struct ethtool_channels *cmd) 377 { 378 struct gve_priv *priv = netdev_priv(netdev); 379 380 cmd->max_rx = priv->rx_cfg.max_queues; 381 cmd->max_tx = priv->tx_cfg.max_queues; 382 cmd->max_other = 0; 383 cmd->max_combined = 0; 384 cmd->rx_count = priv->rx_cfg.num_queues; 385 cmd->tx_count = priv->tx_cfg.num_queues; 386 cmd->other_count = 0; 387 cmd->combined_count = 0; 388 } 389 390 static int gve_set_channels(struct net_device *netdev, 391 struct ethtool_channels *cmd) 392 { 393 struct gve_priv *priv = netdev_priv(netdev); 394 struct gve_queue_config new_tx_cfg = priv->tx_cfg; 395 struct gve_queue_config new_rx_cfg = priv->rx_cfg; 396 struct ethtool_channels old_settings; 397 int new_tx = cmd->tx_count; 398 int new_rx = cmd->rx_count; 399 400 gve_get_channels(netdev, &old_settings); 401 402 /* Changing combined is not allowed */ 403 if (cmd->combined_count != old_settings.combined_count) 404 return -EINVAL; 405 406 if (!new_rx || !new_tx) 407 return -EINVAL; 408 409 if (!netif_carrier_ok(netdev)) { 410 priv->tx_cfg.num_queues = new_tx; 411 priv->rx_cfg.num_queues = new_rx; 412 return 0; 413 } 414 415 new_tx_cfg.num_queues = new_tx; 416 new_rx_cfg.num_queues = new_rx; 417 418 return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg); 419 } 420 421 static void gve_get_ringparam(struct net_device *netdev, 422 struct ethtool_ringparam *cmd) 423 { 424 struct gve_priv *priv = netdev_priv(netdev); 425 426 cmd->rx_max_pending = priv->rx_desc_cnt; 427 cmd->tx_max_pending = priv->tx_desc_cnt; 428 cmd->rx_pending = priv->rx_desc_cnt; 429 cmd->tx_pending = priv->tx_desc_cnt; 430 } 431 432 static int gve_user_reset(struct net_device *netdev, u32 *flags) 433 { 434 struct gve_priv *priv = netdev_priv(netdev); 435 436 if (*flags == ETH_RESET_ALL) { 437 *flags = 0; 438 return gve_reset(priv, true); 439 } 440 441 return -EOPNOTSUPP; 442 } 443 444 static int gve_get_tunable(struct net_device *netdev, 445 const struct ethtool_tunable *etuna, void *value) 446 { 447 struct gve_priv *priv = netdev_priv(netdev); 448 449 switch (etuna->id) { 450 case ETHTOOL_RX_COPYBREAK: 451 *(u32 *)value = priv->rx_copybreak; 452 return 0; 453 default: 454 return -EOPNOTSUPP; 455 } 456 } 457 458 static int gve_set_tunable(struct net_device *netdev, 459 const struct ethtool_tunable *etuna, 460 const void *value) 461 { 462 struct gve_priv *priv = netdev_priv(netdev); 463 u32 len; 464 465 switch (etuna->id) { 466 case ETHTOOL_RX_COPYBREAK: 467 { 468 u32 max_copybreak = gve_is_gqi(priv) ? 469 (PAGE_SIZE / 2) : priv->data_buffer_size_dqo; 470 471 len = *(u32 *)value; 472 if (len > max_copybreak) 473 return -EINVAL; 474 priv->rx_copybreak = len; 475 return 0; 476 } 477 default: 478 return -EOPNOTSUPP; 479 } 480 } 481 482 static u32 gve_get_priv_flags(struct net_device *netdev) 483 { 484 struct gve_priv *priv = netdev_priv(netdev); 485 u32 ret_flags = 0; 486 487 /* Only 1 flag exists currently: report-stats (BIT(O)), so set that flag. */ 488 if (priv->ethtool_flags & BIT(0)) 489 ret_flags |= BIT(0); 490 return ret_flags; 491 } 492 493 static int gve_set_priv_flags(struct net_device *netdev, u32 flags) 494 { 495 struct gve_priv *priv = netdev_priv(netdev); 496 u64 ori_flags, new_flags; 497 498 ori_flags = READ_ONCE(priv->ethtool_flags); 499 new_flags = ori_flags; 500 501 /* Only one priv flag exists: report-stats (BIT(0))*/ 502 if (flags & BIT(0)) 503 new_flags |= BIT(0); 504 else 505 new_flags &= ~(BIT(0)); 506 priv->ethtool_flags = new_flags; 507 /* start report-stats timer when user turns report stats on. */ 508 if (flags & BIT(0)) { 509 mod_timer(&priv->stats_report_timer, 510 round_jiffies(jiffies + 511 msecs_to_jiffies(priv->stats_report_timer_period))); 512 } 513 /* Zero off gve stats when report-stats turned off and */ 514 /* delete report stats timer. */ 515 if (!(flags & BIT(0)) && (ori_flags & BIT(0))) { 516 int tx_stats_num = GVE_TX_STATS_REPORT_NUM * 517 priv->tx_cfg.num_queues; 518 int rx_stats_num = GVE_RX_STATS_REPORT_NUM * 519 priv->rx_cfg.num_queues; 520 521 memset(priv->stats_report->stats, 0, (tx_stats_num + rx_stats_num) * 522 sizeof(struct stats)); 523 del_timer_sync(&priv->stats_report_timer); 524 } 525 return 0; 526 } 527 528 static int gve_get_link_ksettings(struct net_device *netdev, 529 struct ethtool_link_ksettings *cmd) 530 { 531 struct gve_priv *priv = netdev_priv(netdev); 532 int err = gve_adminq_report_link_speed(priv); 533 534 cmd->base.speed = priv->link_speed; 535 return err; 536 } 537 538 const struct ethtool_ops gve_ethtool_ops = { 539 .get_drvinfo = gve_get_drvinfo, 540 .get_strings = gve_get_strings, 541 .get_sset_count = gve_get_sset_count, 542 .get_ethtool_stats = gve_get_ethtool_stats, 543 .set_msglevel = gve_set_msglevel, 544 .get_msglevel = gve_get_msglevel, 545 .set_channels = gve_set_channels, 546 .get_channels = gve_get_channels, 547 .get_link = ethtool_op_get_link, 548 .get_ringparam = gve_get_ringparam, 549 .reset = gve_user_reset, 550 .get_tunable = gve_get_tunable, 551 .set_tunable = gve_set_tunable, 552 .get_priv_flags = gve_get_priv_flags, 553 .set_priv_flags = gve_set_priv_flags, 554 .get_link_ksettings = gve_get_link_ksettings 555 }; 556