1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 * Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #ifndef _GVE_H_ 8 #define _GVE_H_ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/netdevice.h> 12 #include <linux/pci.h> 13 #include <linux/u64_stats_sync.h> 14 15 #include "gve_desc.h" 16 #include "gve_desc_dqo.h" 17 18 #ifndef PCI_VENDOR_ID_GOOGLE 19 #define PCI_VENDOR_ID_GOOGLE 0x1ae0 20 #endif 21 22 #define PCI_DEV_ID_GVNIC 0x0042 23 24 #define GVE_REGISTER_BAR 0 25 #define GVE_DOORBELL_BAR 2 26 27 /* Driver can alloc up to 2 segments for the header and 2 for the payload. */ 28 #define GVE_TX_MAX_IOVEC 4 29 /* 1 for management, 1 for rx, 1 for tx */ 30 #define GVE_MIN_MSIX 3 31 32 /* Numbers of gve tx/rx stats in stats report. */ 33 #define GVE_TX_STATS_REPORT_NUM 6 34 #define GVE_RX_STATS_REPORT_NUM 2 35 36 /* Interval to schedule a stats report update, 20000ms. */ 37 #define GVE_STATS_REPORT_TIMER_PERIOD 20000 38 39 /* Numbers of NIC tx/rx stats in stats report. */ 40 #define NIC_TX_STATS_REPORT_NUM 0 41 #define NIC_RX_STATS_REPORT_NUM 4 42 43 #define GVE_DATA_SLOT_ADDR_PAGE_MASK (~(PAGE_SIZE - 1)) 44 45 /* PTYPEs are always 10 bits. */ 46 #define GVE_NUM_PTYPES 1024 47 48 #define GVE_RX_BUFFER_SIZE_DQO 2048 49 50 /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ 51 struct gve_rx_desc_queue { 52 struct gve_rx_desc *desc_ring; /* the descriptor ring */ 53 dma_addr_t bus; /* the bus for the desc_ring */ 54 u8 seqno; /* the next expected seqno for this desc*/ 55 }; 56 57 /* The page info for a single slot in the RX data queue */ 58 struct gve_rx_slot_page_info { 59 struct page *page; 60 void *page_address; 61 u32 page_offset; /* offset to write to in page */ 62 int pagecnt_bias; /* expected pagecnt if only the driver has a ref */ 63 u8 can_flip; 64 }; 65 66 /* A list of pages registered with the device during setup and used by a queue 67 * as buffers 68 */ 69 struct gve_queue_page_list { 70 u32 id; /* unique id */ 71 u32 num_entries; 72 struct page **pages; /* list of num_entries pages */ 73 dma_addr_t *page_buses; /* the dma addrs of the pages */ 74 }; 75 76 /* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */ 77 struct gve_rx_data_queue { 78 union gve_rx_data_slot *data_ring; /* read by NIC */ 79 dma_addr_t data_bus; /* dma mapping of the slots */ 80 struct gve_rx_slot_page_info *page_info; /* page info of the buffers */ 81 struct gve_queue_page_list *qpl; /* qpl assigned to this queue */ 82 u8 raw_addressing; /* use raw_addressing? */ 83 }; 84 85 struct gve_priv; 86 87 /* RX buffer queue for posting buffers to HW. 88 * Each RX (completion) queue has a corresponding buffer queue. 89 */ 90 struct gve_rx_buf_queue_dqo { 91 struct gve_rx_desc_dqo *desc_ring; 92 dma_addr_t bus; 93 u32 head; /* Pointer to start cleaning buffers at. */ 94 u32 tail; /* Last posted buffer index + 1 */ 95 u32 mask; /* Mask for indices to the size of the ring */ 96 }; 97 98 /* RX completion queue to receive packets from HW. */ 99 struct gve_rx_compl_queue_dqo { 100 struct gve_rx_compl_desc_dqo *desc_ring; 101 dma_addr_t bus; 102 103 /* Number of slots which did not have a buffer posted yet. We should not 104 * post more buffers than the queue size to avoid HW overrunning the 105 * queue. 106 */ 107 int num_free_slots; 108 109 /* HW uses a "generation bit" to notify SW of new descriptors. When a 110 * descriptor's generation bit is different from the current generation, 111 * that descriptor is ready to be consumed by SW. 112 */ 113 u8 cur_gen_bit; 114 115 /* Pointer into desc_ring where the next completion descriptor will be 116 * received. 117 */ 118 u32 head; 119 u32 mask; /* Mask for indices to the size of the ring */ 120 }; 121 122 /* Stores state for tracking buffers posted to HW */ 123 struct gve_rx_buf_state_dqo { 124 /* The page posted to HW. */ 125 struct gve_rx_slot_page_info page_info; 126 127 /* The DMA address corresponding to `page_info`. */ 128 dma_addr_t addr; 129 130 /* Last offset into the page when it only had a single reference, at 131 * which point every other offset is free to be reused. 132 */ 133 u32 last_single_ref_offset; 134 135 /* Linked list index to next element in the list, or -1 if none */ 136 s16 next; 137 }; 138 139 /* `head` and `tail` are indices into an array, or -1 if empty. */ 140 struct gve_index_list { 141 s16 head; 142 s16 tail; 143 }; 144 145 /* A single received packet split across multiple buffers may be 146 * reconstructed using the information in this structure. 147 */ 148 struct gve_rx_ctx { 149 /* head and tail of skb chain for the current packet or NULL if none */ 150 struct sk_buff *skb_head; 151 struct sk_buff *skb_tail; 152 u16 total_expected_size; 153 u8 expected_frag_cnt; 154 u8 curr_frag_cnt; 155 u8 reuse_frags; 156 }; 157 158 /* Contains datapath state used to represent an RX queue. */ 159 struct gve_rx_ring { 160 struct gve_priv *gve; 161 union { 162 /* GQI fields */ 163 struct { 164 struct gve_rx_desc_queue desc; 165 struct gve_rx_data_queue data; 166 167 /* threshold for posting new buffs and descs */ 168 u32 db_threshold; 169 u16 packet_buffer_size; 170 }; 171 172 /* DQO fields. */ 173 struct { 174 struct gve_rx_buf_queue_dqo bufq; 175 struct gve_rx_compl_queue_dqo complq; 176 177 struct gve_rx_buf_state_dqo *buf_states; 178 u16 num_buf_states; 179 180 /* Linked list of gve_rx_buf_state_dqo. Index into 181 * buf_states, or -1 if empty. 182 */ 183 s16 free_buf_states; 184 185 /* Linked list of gve_rx_buf_state_dqo. Indexes into 186 * buf_states, or -1 if empty. 187 * 188 * This list contains buf_states which are pointing to 189 * valid buffers. 190 * 191 * We use a FIFO here in order to increase the 192 * probability that buffers can be reused by increasing 193 * the time between usages. 194 */ 195 struct gve_index_list recycled_buf_states; 196 197 /* Linked list of gve_rx_buf_state_dqo. Indexes into 198 * buf_states, or -1 if empty. 199 * 200 * This list contains buf_states which have buffers 201 * which cannot be reused yet. 202 */ 203 struct gve_index_list used_buf_states; 204 } dqo; 205 }; 206 207 u64 rbytes; /* free-running bytes received */ 208 u64 rpackets; /* free-running packets received */ 209 u32 cnt; /* free-running total number of completed packets */ 210 u32 fill_cnt; /* free-running total number of descs and buffs posted */ 211 u32 mask; /* masks the cnt and fill_cnt to the size of the ring */ 212 u64 rx_copybreak_pkt; /* free-running count of copybreak packets */ 213 u64 rx_copied_pkt; /* free-running total number of copied packets */ 214 u64 rx_skb_alloc_fail; /* free-running count of skb alloc fails */ 215 u64 rx_buf_alloc_fail; /* free-running count of buffer alloc fails */ 216 u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */ 217 u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */ 218 u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */ 219 u64 rx_frag_copy_cnt; /* free-running count of rx segments copied into skb linear portion */ 220 u32 q_num; /* queue index */ 221 u32 ntfy_id; /* notification block index */ 222 struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 223 dma_addr_t q_resources_bus; /* dma address for the queue resources */ 224 struct u64_stats_sync statss; /* sync stats for 32bit archs */ 225 226 struct gve_rx_ctx ctx; /* Info for packet currently being processed in this ring. */ 227 }; 228 229 /* A TX desc ring entry */ 230 union gve_tx_desc { 231 struct gve_tx_pkt_desc pkt; /* first desc for a packet */ 232 struct gve_tx_mtd_desc mtd; /* optional metadata descriptor */ 233 struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ 234 }; 235 236 /* Tracks the memory in the fifo occupied by a segment of a packet */ 237 struct gve_tx_iovec { 238 u32 iov_offset; /* offset into this segment */ 239 u32 iov_len; /* length */ 240 u32 iov_padding; /* padding associated with this segment */ 241 }; 242 243 /* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc 244 * ring entry but only used for a pkt_desc not a seg_desc 245 */ 246 struct gve_tx_buffer_state { 247 struct sk_buff *skb; /* skb for this pkt */ 248 union { 249 struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */ 250 struct { 251 DEFINE_DMA_UNMAP_ADDR(dma); 252 DEFINE_DMA_UNMAP_LEN(len); 253 }; 254 }; 255 }; 256 257 /* A TX buffer - each queue has one */ 258 struct gve_tx_fifo { 259 void *base; /* address of base of FIFO */ 260 u32 size; /* total size */ 261 atomic_t available; /* how much space is still available */ 262 u32 head; /* offset to write at */ 263 struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */ 264 }; 265 266 /* TX descriptor for DQO format */ 267 union gve_tx_desc_dqo { 268 struct gve_tx_pkt_desc_dqo pkt; 269 struct gve_tx_tso_context_desc_dqo tso_ctx; 270 struct gve_tx_general_context_desc_dqo general_ctx; 271 }; 272 273 enum gve_packet_state { 274 /* Packet is in free list, available to be allocated. 275 * This should always be zero since state is not explicitly initialized. 276 */ 277 GVE_PACKET_STATE_UNALLOCATED, 278 /* Packet is expecting a regular data completion or miss completion */ 279 GVE_PACKET_STATE_PENDING_DATA_COMPL, 280 /* Packet has received a miss completion and is expecting a 281 * re-injection completion. 282 */ 283 GVE_PACKET_STATE_PENDING_REINJECT_COMPL, 284 /* No valid completion received within the specified timeout. */ 285 GVE_PACKET_STATE_TIMED_OUT_COMPL, 286 }; 287 288 struct gve_tx_pending_packet_dqo { 289 struct sk_buff *skb; /* skb for this packet */ 290 291 /* 0th element corresponds to the linear portion of `skb`, should be 292 * unmapped with `dma_unmap_single`. 293 * 294 * All others correspond to `skb`'s frags and should be unmapped with 295 * `dma_unmap_page`. 296 */ 297 DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]); 298 DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]); 299 u16 num_bufs; 300 301 /* Linked list index to next element in the list, or -1 if none */ 302 s16 next; 303 304 /* Linked list index to prev element in the list, or -1 if none. 305 * Used for tracking either outstanding miss completions or prematurely 306 * freed packets. 307 */ 308 s16 prev; 309 310 /* Identifies the current state of the packet as defined in 311 * `enum gve_packet_state`. 312 */ 313 u8 state; 314 315 /* If packet is an outstanding miss completion, then the packet is 316 * freed if the corresponding re-injection completion is not received 317 * before kernel jiffies exceeds timeout_jiffies. 318 */ 319 unsigned long timeout_jiffies; 320 }; 321 322 /* Contains datapath state used to represent a TX queue. */ 323 struct gve_tx_ring { 324 /* Cacheline 0 -- Accessed & dirtied during transmit */ 325 union { 326 /* GQI fields */ 327 struct { 328 struct gve_tx_fifo tx_fifo; 329 u32 req; /* driver tracked head pointer */ 330 u32 done; /* driver tracked tail pointer */ 331 }; 332 333 /* DQO fields. */ 334 struct { 335 /* Linked list of gve_tx_pending_packet_dqo. Index into 336 * pending_packets, or -1 if empty. 337 * 338 * This is a consumer list owned by the TX path. When it 339 * runs out, the producer list is stolen from the 340 * completion handling path 341 * (dqo_compl.free_pending_packets). 342 */ 343 s16 free_pending_packets; 344 345 /* Cached value of `dqo_compl.hw_tx_head` */ 346 u32 head; 347 u32 tail; /* Last posted buffer index + 1 */ 348 349 /* Index of the last descriptor with "report event" bit 350 * set. 351 */ 352 u32 last_re_idx; 353 } dqo_tx; 354 }; 355 356 /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */ 357 union { 358 /* GQI fields */ 359 struct { 360 /* Spinlock for when cleanup in progress */ 361 spinlock_t clean_lock; 362 }; 363 364 /* DQO fields. */ 365 struct { 366 u32 head; /* Last read on compl_desc */ 367 368 /* Tracks the current gen bit of compl_q */ 369 u8 cur_gen_bit; 370 371 /* Linked list of gve_tx_pending_packet_dqo. Index into 372 * pending_packets, or -1 if empty. 373 * 374 * This is the producer list, owned by the completion 375 * handling path. When the consumer list 376 * (dqo_tx.free_pending_packets) is runs out, this list 377 * will be stolen. 378 */ 379 atomic_t free_pending_packets; 380 381 /* Last TX ring index fetched by HW */ 382 atomic_t hw_tx_head; 383 384 /* List to track pending packets which received a miss 385 * completion but not a corresponding reinjection. 386 */ 387 struct gve_index_list miss_completions; 388 389 /* List to track pending packets that were completed 390 * before receiving a valid completion because they 391 * reached a specified timeout. 392 */ 393 struct gve_index_list timed_out_completions; 394 } dqo_compl; 395 } ____cacheline_aligned; 396 u64 pkt_done; /* free-running - total packets completed */ 397 u64 bytes_done; /* free-running - total bytes completed */ 398 u64 dropped_pkt; /* free-running - total packets dropped */ 399 u64 dma_mapping_error; /* count of dma mapping errors */ 400 401 /* Cacheline 2 -- Read-mostly fields */ 402 union { 403 /* GQI fields */ 404 struct { 405 union gve_tx_desc *desc; 406 407 /* Maps 1:1 to a desc */ 408 struct gve_tx_buffer_state *info; 409 }; 410 411 /* DQO fields. */ 412 struct { 413 union gve_tx_desc_dqo *tx_ring; 414 struct gve_tx_compl_desc *compl_ring; 415 416 struct gve_tx_pending_packet_dqo *pending_packets; 417 s16 num_pending_packets; 418 419 u32 complq_mask; /* complq size is complq_mask + 1 */ 420 } dqo; 421 } ____cacheline_aligned; 422 struct netdev_queue *netdev_txq; 423 struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 424 struct device *dev; 425 u32 mask; /* masks req and done down to queue size */ 426 u8 raw_addressing; /* use raw_addressing? */ 427 428 /* Slow-path fields */ 429 u32 q_num ____cacheline_aligned; /* queue idx */ 430 u32 stop_queue; /* count of queue stops */ 431 u32 wake_queue; /* count of queue wakes */ 432 u32 queue_timeout; /* count of queue timeouts */ 433 u32 ntfy_id; /* notification block index */ 434 u32 last_kick_msec; /* Last time the queue was kicked */ 435 dma_addr_t bus; /* dma address of the descr ring */ 436 dma_addr_t q_resources_bus; /* dma address of the queue resources */ 437 dma_addr_t complq_bus_dqo; /* dma address of the dqo.compl_ring */ 438 struct u64_stats_sync statss; /* sync stats for 32bit archs */ 439 } ____cacheline_aligned; 440 441 /* Wraps the info for one irq including the napi struct and the queues 442 * associated with that irq. 443 */ 444 struct gve_notify_block { 445 __be32 *irq_db_index; /* pointer to idx into Bar2 */ 446 char name[IFNAMSIZ + 16]; /* name registered with the kernel */ 447 struct napi_struct napi; /* kernel napi struct for this block */ 448 struct gve_priv *priv; 449 struct gve_tx_ring *tx; /* tx rings on this block */ 450 struct gve_rx_ring *rx; /* rx rings on this block */ 451 }; 452 453 /* Tracks allowed and current queue settings */ 454 struct gve_queue_config { 455 u16 max_queues; 456 u16 num_queues; /* current */ 457 }; 458 459 /* Tracks the available and used qpl IDs */ 460 struct gve_qpl_config { 461 u32 qpl_map_size; /* map memory size */ 462 unsigned long *qpl_id_map; /* bitmap of used qpl ids */ 463 }; 464 465 struct gve_options_dqo_rda { 466 u16 tx_comp_ring_entries; /* number of tx_comp descriptors */ 467 u16 rx_buff_ring_entries; /* number of rx_buff descriptors */ 468 }; 469 470 struct gve_irq_db { 471 __be32 index; 472 } ____cacheline_aligned; 473 474 struct gve_ptype { 475 u8 l3_type; /* `gve_l3_type` in gve_adminq.h */ 476 u8 l4_type; /* `gve_l4_type` in gve_adminq.h */ 477 }; 478 479 struct gve_ptype_lut { 480 struct gve_ptype ptypes[GVE_NUM_PTYPES]; 481 }; 482 483 /* GVE_QUEUE_FORMAT_UNSPECIFIED must be zero since 0 is the default value 484 * when the entire configure_device_resources command is zeroed out and the 485 * queue_format is not specified. 486 */ 487 enum gve_queue_format { 488 GVE_QUEUE_FORMAT_UNSPECIFIED = 0x0, 489 GVE_GQI_RDA_FORMAT = 0x1, 490 GVE_GQI_QPL_FORMAT = 0x2, 491 GVE_DQO_RDA_FORMAT = 0x3, 492 }; 493 494 struct gve_priv { 495 struct net_device *dev; 496 struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ 497 struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */ 498 struct gve_queue_page_list *qpls; /* array of num qpls */ 499 struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */ 500 struct gve_irq_db *irq_db_indices; /* array of num_ntfy_blks */ 501 dma_addr_t irq_db_indices_bus; 502 struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */ 503 char mgmt_msix_name[IFNAMSIZ + 16]; 504 u32 mgmt_msix_idx; 505 __be32 *counter_array; /* array of num_event_counters */ 506 dma_addr_t counter_array_bus; 507 508 u16 num_event_counters; 509 u16 tx_desc_cnt; /* num desc per ring */ 510 u16 rx_desc_cnt; /* num desc per ring */ 511 u16 tx_pages_per_qpl; /* tx buffer length */ 512 u16 rx_data_slot_cnt; /* rx buffer length */ 513 u64 max_registered_pages; 514 u64 num_registered_pages; /* num pages registered with NIC */ 515 u32 rx_copybreak; /* copy packets smaller than this */ 516 u16 default_num_queues; /* default num queues to set up */ 517 518 struct gve_queue_config tx_cfg; 519 struct gve_queue_config rx_cfg; 520 struct gve_qpl_config qpl_cfg; /* map used QPL ids */ 521 u32 num_ntfy_blks; /* spilt between TX and RX so must be even */ 522 523 struct gve_registers __iomem *reg_bar0; /* see gve_register.h */ 524 __be32 __iomem *db_bar2; /* "array" of doorbells */ 525 u32 msg_enable; /* level for netif* netdev print macros */ 526 struct pci_dev *pdev; 527 528 /* metrics */ 529 u32 tx_timeo_cnt; 530 531 /* Admin queue - see gve_adminq.h*/ 532 union gve_adminq_command *adminq; 533 dma_addr_t adminq_bus_addr; 534 u32 adminq_mask; /* masks prod_cnt to adminq size */ 535 u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */ 536 u32 adminq_cmd_fail; /* free-running count of AQ cmds failed */ 537 u32 adminq_timeouts; /* free-running count of AQ cmds timeouts */ 538 /* free-running count of per AQ cmd executed */ 539 u32 adminq_describe_device_cnt; 540 u32 adminq_cfg_device_resources_cnt; 541 u32 adminq_register_page_list_cnt; 542 u32 adminq_unregister_page_list_cnt; 543 u32 adminq_create_tx_queue_cnt; 544 u32 adminq_create_rx_queue_cnt; 545 u32 adminq_destroy_tx_queue_cnt; 546 u32 adminq_destroy_rx_queue_cnt; 547 u32 adminq_dcfg_device_resources_cnt; 548 u32 adminq_set_driver_parameter_cnt; 549 u32 adminq_report_stats_cnt; 550 u32 adminq_report_link_speed_cnt; 551 u32 adminq_get_ptype_map_cnt; 552 553 /* Global stats */ 554 u32 interface_up_cnt; /* count of times interface turned up since last reset */ 555 u32 interface_down_cnt; /* count of times interface turned down since last reset */ 556 u32 reset_cnt; /* count of reset */ 557 u32 page_alloc_fail; /* count of page alloc fails */ 558 u32 dma_mapping_error; /* count of dma mapping errors */ 559 u32 stats_report_trigger_cnt; /* count of device-requested stats-reports since last reset */ 560 u32 suspend_cnt; /* count of times suspended */ 561 u32 resume_cnt; /* count of times resumed */ 562 struct workqueue_struct *gve_wq; 563 struct work_struct service_task; 564 struct work_struct stats_report_task; 565 unsigned long service_task_flags; 566 unsigned long state_flags; 567 568 struct gve_stats_report *stats_report; 569 u64 stats_report_len; 570 dma_addr_t stats_report_bus; /* dma address for the stats report */ 571 unsigned long ethtool_flags; 572 573 unsigned long stats_report_timer_period; 574 struct timer_list stats_report_timer; 575 576 /* Gvnic device link speed from hypervisor. */ 577 u64 link_speed; 578 bool up_before_suspend; /* True if dev was up before suspend */ 579 580 struct gve_options_dqo_rda options_dqo_rda; 581 struct gve_ptype_lut *ptype_lut_dqo; 582 583 /* Must be a power of two. */ 584 int data_buffer_size_dqo; 585 586 enum gve_queue_format queue_format; 587 588 /* Interrupt coalescing settings */ 589 u32 tx_coalesce_usecs; 590 u32 rx_coalesce_usecs; 591 }; 592 593 enum gve_service_task_flags_bit { 594 GVE_PRIV_FLAGS_DO_RESET = 1, 595 GVE_PRIV_FLAGS_RESET_IN_PROGRESS = 2, 596 GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = 3, 597 GVE_PRIV_FLAGS_DO_REPORT_STATS = 4, 598 }; 599 600 enum gve_state_flags_bit { 601 GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = 1, 602 GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = 2, 603 GVE_PRIV_FLAGS_DEVICE_RINGS_OK = 3, 604 GVE_PRIV_FLAGS_NAPI_ENABLED = 4, 605 }; 606 607 enum gve_ethtool_flags_bit { 608 GVE_PRIV_FLAGS_REPORT_STATS = 0, 609 }; 610 611 static inline bool gve_get_do_reset(struct gve_priv *priv) 612 { 613 return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 614 } 615 616 static inline void gve_set_do_reset(struct gve_priv *priv) 617 { 618 set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 619 } 620 621 static inline void gve_clear_do_reset(struct gve_priv *priv) 622 { 623 clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 624 } 625 626 static inline bool gve_get_reset_in_progress(struct gve_priv *priv) 627 { 628 return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, 629 &priv->service_task_flags); 630 } 631 632 static inline void gve_set_reset_in_progress(struct gve_priv *priv) 633 { 634 set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 635 } 636 637 static inline void gve_clear_reset_in_progress(struct gve_priv *priv) 638 { 639 clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 640 } 641 642 static inline bool gve_get_probe_in_progress(struct gve_priv *priv) 643 { 644 return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, 645 &priv->service_task_flags); 646 } 647 648 static inline void gve_set_probe_in_progress(struct gve_priv *priv) 649 { 650 set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 651 } 652 653 static inline void gve_clear_probe_in_progress(struct gve_priv *priv) 654 { 655 clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 656 } 657 658 static inline bool gve_get_do_report_stats(struct gve_priv *priv) 659 { 660 return test_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, 661 &priv->service_task_flags); 662 } 663 664 static inline void gve_set_do_report_stats(struct gve_priv *priv) 665 { 666 set_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags); 667 } 668 669 static inline void gve_clear_do_report_stats(struct gve_priv *priv) 670 { 671 clear_bit(GVE_PRIV_FLAGS_DO_REPORT_STATS, &priv->service_task_flags); 672 } 673 674 static inline bool gve_get_admin_queue_ok(struct gve_priv *priv) 675 { 676 return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 677 } 678 679 static inline void gve_set_admin_queue_ok(struct gve_priv *priv) 680 { 681 set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 682 } 683 684 static inline void gve_clear_admin_queue_ok(struct gve_priv *priv) 685 { 686 clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 687 } 688 689 static inline bool gve_get_device_resources_ok(struct gve_priv *priv) 690 { 691 return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 692 } 693 694 static inline void gve_set_device_resources_ok(struct gve_priv *priv) 695 { 696 set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 697 } 698 699 static inline void gve_clear_device_resources_ok(struct gve_priv *priv) 700 { 701 clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 702 } 703 704 static inline bool gve_get_device_rings_ok(struct gve_priv *priv) 705 { 706 return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 707 } 708 709 static inline void gve_set_device_rings_ok(struct gve_priv *priv) 710 { 711 set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 712 } 713 714 static inline void gve_clear_device_rings_ok(struct gve_priv *priv) 715 { 716 clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 717 } 718 719 static inline bool gve_get_napi_enabled(struct gve_priv *priv) 720 { 721 return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 722 } 723 724 static inline void gve_set_napi_enabled(struct gve_priv *priv) 725 { 726 set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 727 } 728 729 static inline void gve_clear_napi_enabled(struct gve_priv *priv) 730 { 731 clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 732 } 733 734 static inline bool gve_get_report_stats(struct gve_priv *priv) 735 { 736 return test_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags); 737 } 738 739 static inline void gve_clear_report_stats(struct gve_priv *priv) 740 { 741 clear_bit(GVE_PRIV_FLAGS_REPORT_STATS, &priv->ethtool_flags); 742 } 743 744 /* Returns the address of the ntfy_blocks irq doorbell 745 */ 746 static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv, 747 struct gve_notify_block *block) 748 { 749 return &priv->db_bar2[be32_to_cpu(*block->irq_db_index)]; 750 } 751 752 /* Returns the index into ntfy_blocks of the given tx ring's block 753 */ 754 static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 755 { 756 return queue_idx; 757 } 758 759 /* Returns the index into ntfy_blocks of the given rx ring's block 760 */ 761 static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 762 { 763 return (priv->num_ntfy_blks / 2) + queue_idx; 764 } 765 766 /* Returns the number of tx queue page lists 767 */ 768 static inline u32 gve_num_tx_qpls(struct gve_priv *priv) 769 { 770 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 771 return 0; 772 773 return priv->tx_cfg.num_queues; 774 } 775 776 /* Returns the number of rx queue page lists 777 */ 778 static inline u32 gve_num_rx_qpls(struct gve_priv *priv) 779 { 780 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 781 return 0; 782 783 return priv->rx_cfg.num_queues; 784 } 785 786 /* Returns a pointer to the next available tx qpl in the list of qpls 787 */ 788 static inline 789 struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv) 790 { 791 int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map, 792 priv->qpl_cfg.qpl_map_size); 793 794 /* we are out of tx qpls */ 795 if (id >= gve_num_tx_qpls(priv)) 796 return NULL; 797 798 set_bit(id, priv->qpl_cfg.qpl_id_map); 799 return &priv->qpls[id]; 800 } 801 802 /* Returns a pointer to the next available rx qpl in the list of qpls 803 */ 804 static inline 805 struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv) 806 { 807 int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map, 808 priv->qpl_cfg.qpl_map_size, 809 gve_num_tx_qpls(priv)); 810 811 /* we are out of rx qpls */ 812 if (id == gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv)) 813 return NULL; 814 815 set_bit(id, priv->qpl_cfg.qpl_id_map); 816 return &priv->qpls[id]; 817 } 818 819 /* Unassigns the qpl with the given id 820 */ 821 static inline void gve_unassign_qpl(struct gve_priv *priv, int id) 822 { 823 clear_bit(id, priv->qpl_cfg.qpl_id_map); 824 } 825 826 /* Returns the correct dma direction for tx and rx qpls 827 */ 828 static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, 829 int id) 830 { 831 if (id < gve_num_tx_qpls(priv)) 832 return DMA_TO_DEVICE; 833 else 834 return DMA_FROM_DEVICE; 835 } 836 837 static inline bool gve_is_gqi(struct gve_priv *priv) 838 { 839 return priv->queue_format == GVE_GQI_RDA_FORMAT || 840 priv->queue_format == GVE_GQI_QPL_FORMAT; 841 } 842 843 /* buffers */ 844 int gve_alloc_page(struct gve_priv *priv, struct device *dev, 845 struct page **page, dma_addr_t *dma, 846 enum dma_data_direction); 847 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 848 enum dma_data_direction); 849 /* tx handling */ 850 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev); 851 bool gve_tx_poll(struct gve_notify_block *block, int budget); 852 int gve_tx_alloc_rings(struct gve_priv *priv); 853 void gve_tx_free_rings_gqi(struct gve_priv *priv); 854 u32 gve_tx_load_event_counter(struct gve_priv *priv, 855 struct gve_tx_ring *tx); 856 bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx); 857 /* rx handling */ 858 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx); 859 int gve_rx_poll(struct gve_notify_block *block, int budget); 860 bool gve_rx_work_pending(struct gve_rx_ring *rx); 861 int gve_rx_alloc_rings(struct gve_priv *priv); 862 void gve_rx_free_rings_gqi(struct gve_priv *priv); 863 /* Reset */ 864 void gve_schedule_reset(struct gve_priv *priv); 865 int gve_reset(struct gve_priv *priv, bool attempt_teardown); 866 int gve_adjust_queues(struct gve_priv *priv, 867 struct gve_queue_config new_rx_config, 868 struct gve_queue_config new_tx_config); 869 /* report stats handling */ 870 void gve_handle_report_stats(struct gve_priv *priv); 871 /* exported by ethtool.c */ 872 extern const struct ethtool_ops gve_ethtool_ops; 873 /* needed by ethtool */ 874 extern const char gve_version_str[]; 875 #endif /* _GVE_H_ */ 876