1 /* SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 * Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2019 Google, Inc. 5 */ 6 7 #ifndef _GVE_H_ 8 #define _GVE_H_ 9 10 #include <linux/dma-mapping.h> 11 #include <linux/netdevice.h> 12 #include <linux/pci.h> 13 #include <linux/u64_stats_sync.h> 14 #include "gve_desc.h" 15 16 #ifndef PCI_VENDOR_ID_GOOGLE 17 #define PCI_VENDOR_ID_GOOGLE 0x1ae0 18 #endif 19 20 #define PCI_DEV_ID_GVNIC 0x0042 21 22 #define GVE_REGISTER_BAR 0 23 #define GVE_DOORBELL_BAR 2 24 25 /* Driver can alloc up to 2 segments for the header and 2 for the payload. */ 26 #define GVE_TX_MAX_IOVEC 4 27 /* 1 for management, 1 for rx, 1 for tx */ 28 #define GVE_MIN_MSIX 3 29 30 /* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */ 31 struct gve_rx_desc_queue { 32 struct gve_rx_desc *desc_ring; /* the descriptor ring */ 33 dma_addr_t bus; /* the bus for the desc_ring */ 34 u32 cnt; /* free-running total number of completed packets */ 35 u32 fill_cnt; /* free-running total number of descriptors posted */ 36 u32 mask; /* masks the cnt to the size of the ring */ 37 u8 seqno; /* the next expected seqno for this desc*/ 38 }; 39 40 /* The page info for a single slot in the RX data queue */ 41 struct gve_rx_slot_page_info { 42 struct page *page; 43 void *page_address; 44 u32 page_offset; /* offset to write to in page */ 45 }; 46 47 /* A list of pages registered with the device during setup and used by a queue 48 * as buffers 49 */ 50 struct gve_queue_page_list { 51 u32 id; /* unique id */ 52 u32 num_entries; 53 struct page **pages; /* list of num_entries pages */ 54 dma_addr_t *page_buses; /* the dma addrs of the pages */ 55 }; 56 57 /* Each slot in the data ring has a 1:1 mapping to a slot in the desc ring */ 58 struct gve_rx_data_queue { 59 struct gve_rx_data_slot *data_ring; /* read by NIC */ 60 dma_addr_t data_bus; /* dma mapping of the slots */ 61 struct gve_rx_slot_page_info *page_info; /* page info of the buffers */ 62 struct gve_queue_page_list *qpl; /* qpl assigned to this queue */ 63 u32 mask; /* masks the cnt to the size of the ring */ 64 u32 cnt; /* free-running total number of completed packets */ 65 }; 66 67 struct gve_priv; 68 69 /* An RX ring that contains a power-of-two sized desc and data ring. */ 70 struct gve_rx_ring { 71 struct gve_priv *gve; 72 struct gve_rx_desc_queue desc; 73 struct gve_rx_data_queue data; 74 u64 rbytes; /* free-running bytes received */ 75 u64 rpackets; /* free-running packets received */ 76 u32 q_num; /* queue index */ 77 u32 ntfy_id; /* notification block index */ 78 struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 79 dma_addr_t q_resources_bus; /* dma address for the queue resources */ 80 struct u64_stats_sync statss; /* sync stats for 32bit archs */ 81 }; 82 83 /* A TX desc ring entry */ 84 union gve_tx_desc { 85 struct gve_tx_pkt_desc pkt; /* first desc for a packet */ 86 struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ 87 }; 88 89 /* Tracks the memory in the fifo occupied by a segment of a packet */ 90 struct gve_tx_iovec { 91 u32 iov_offset; /* offset into this segment */ 92 u32 iov_len; /* length */ 93 u32 iov_padding; /* padding associated with this segment */ 94 }; 95 96 /* Tracks the memory in the fifo occupied by the skb. Mapped 1:1 to a desc 97 * ring entry but only used for a pkt_desc not a seg_desc 98 */ 99 struct gve_tx_buffer_state { 100 struct sk_buff *skb; /* skb for this pkt */ 101 struct gve_tx_iovec iov[GVE_TX_MAX_IOVEC]; /* segments of this pkt */ 102 }; 103 104 /* A TX buffer - each queue has one */ 105 struct gve_tx_fifo { 106 void *base; /* address of base of FIFO */ 107 u32 size; /* total size */ 108 atomic_t available; /* how much space is still available */ 109 u32 head; /* offset to write at */ 110 struct gve_queue_page_list *qpl; /* QPL mapped into this FIFO */ 111 }; 112 113 /* A TX ring that contains a power-of-two sized desc ring and a FIFO buffer */ 114 struct gve_tx_ring { 115 /* Cacheline 0 -- Accessed & dirtied during transmit */ 116 struct gve_tx_fifo tx_fifo; 117 u32 req; /* driver tracked head pointer */ 118 u32 done; /* driver tracked tail pointer */ 119 120 /* Cacheline 1 -- Accessed & dirtied during gve_clean_tx_done */ 121 __be32 last_nic_done ____cacheline_aligned; /* NIC tail pointer */ 122 u64 pkt_done; /* free-running - total packets completed */ 123 u64 bytes_done; /* free-running - total bytes completed */ 124 125 /* Cacheline 2 -- Read-mostly fields */ 126 union gve_tx_desc *desc ____cacheline_aligned; 127 struct gve_tx_buffer_state *info; /* Maps 1:1 to a desc */ 128 struct netdev_queue *netdev_txq; 129 struct gve_queue_resources *q_resources; /* head and tail pointer idx */ 130 u32 mask; /* masks req and done down to queue size */ 131 132 /* Slow-path fields */ 133 u32 q_num ____cacheline_aligned; /* queue idx */ 134 u32 stop_queue; /* count of queue stops */ 135 u32 wake_queue; /* count of queue wakes */ 136 u32 ntfy_id; /* notification block index */ 137 dma_addr_t bus; /* dma address of the descr ring */ 138 dma_addr_t q_resources_bus; /* dma address of the queue resources */ 139 struct u64_stats_sync statss; /* sync stats for 32bit archs */ 140 } ____cacheline_aligned; 141 142 /* Wraps the info for one irq including the napi struct and the queues 143 * associated with that irq. 144 */ 145 struct gve_notify_block { 146 __be32 irq_db_index; /* idx into Bar2 - set by device, must be 1st */ 147 char name[IFNAMSIZ + 16]; /* name registered with the kernel */ 148 struct napi_struct napi; /* kernel napi struct for this block */ 149 struct gve_priv *priv; 150 struct gve_tx_ring *tx; /* tx rings on this block */ 151 struct gve_rx_ring *rx; /* rx rings on this block */ 152 } ____cacheline_aligned; 153 154 /* Tracks allowed and current queue settings */ 155 struct gve_queue_config { 156 u16 max_queues; 157 u16 num_queues; /* current */ 158 }; 159 160 /* Tracks the available and used qpl IDs */ 161 struct gve_qpl_config { 162 u32 qpl_map_size; /* map memory size */ 163 unsigned long *qpl_id_map; /* bitmap of used qpl ids */ 164 }; 165 166 struct gve_priv { 167 struct net_device *dev; 168 struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ 169 struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */ 170 struct gve_queue_page_list *qpls; /* array of num qpls */ 171 struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */ 172 dma_addr_t ntfy_block_bus; 173 struct msix_entry *msix_vectors; /* array of num_ntfy_blks + 1 */ 174 char mgmt_msix_name[IFNAMSIZ + 16]; 175 u32 mgmt_msix_idx; 176 __be32 *counter_array; /* array of num_event_counters */ 177 dma_addr_t counter_array_bus; 178 179 u16 num_event_counters; 180 u16 tx_desc_cnt; /* num desc per ring */ 181 u16 rx_desc_cnt; /* num desc per ring */ 182 u16 tx_pages_per_qpl; /* tx buffer length */ 183 u16 rx_pages_per_qpl; /* rx buffer length */ 184 u64 max_registered_pages; 185 u64 num_registered_pages; /* num pages registered with NIC */ 186 u32 rx_copybreak; /* copy packets smaller than this */ 187 u16 default_num_queues; /* default num queues to set up */ 188 189 struct gve_queue_config tx_cfg; 190 struct gve_queue_config rx_cfg; 191 struct gve_qpl_config qpl_cfg; /* map used QPL ids */ 192 u32 num_ntfy_blks; /* spilt between TX and RX so must be even */ 193 194 struct gve_registers __iomem *reg_bar0; /* see gve_register.h */ 195 __be32 __iomem *db_bar2; /* "array" of doorbells */ 196 u32 msg_enable; /* level for netif* netdev print macros */ 197 struct pci_dev *pdev; 198 199 /* metrics */ 200 u32 tx_timeo_cnt; 201 202 /* Admin queue - see gve_adminq.h*/ 203 union gve_adminq_command *adminq; 204 dma_addr_t adminq_bus_addr; 205 u32 adminq_mask; /* masks prod_cnt to adminq size */ 206 u32 adminq_prod_cnt; /* free-running count of AQ cmds executed */ 207 208 struct workqueue_struct *gve_wq; 209 struct work_struct service_task; 210 unsigned long service_task_flags; 211 unsigned long state_flags; 212 }; 213 214 enum gve_service_task_flags { 215 GVE_PRIV_FLAGS_DO_RESET = BIT(1), 216 GVE_PRIV_FLAGS_RESET_IN_PROGRESS = BIT(2), 217 GVE_PRIV_FLAGS_PROBE_IN_PROGRESS = BIT(3), 218 }; 219 220 enum gve_state_flags { 221 GVE_PRIV_FLAGS_ADMIN_QUEUE_OK = BIT(1), 222 GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK = BIT(2), 223 GVE_PRIV_FLAGS_DEVICE_RINGS_OK = BIT(3), 224 GVE_PRIV_FLAGS_NAPI_ENABLED = BIT(4), 225 }; 226 227 static inline bool gve_get_do_reset(struct gve_priv *priv) 228 { 229 return test_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 230 } 231 232 static inline void gve_set_do_reset(struct gve_priv *priv) 233 { 234 set_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 235 } 236 237 static inline void gve_clear_do_reset(struct gve_priv *priv) 238 { 239 clear_bit(GVE_PRIV_FLAGS_DO_RESET, &priv->service_task_flags); 240 } 241 242 static inline bool gve_get_reset_in_progress(struct gve_priv *priv) 243 { 244 return test_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, 245 &priv->service_task_flags); 246 } 247 248 static inline void gve_set_reset_in_progress(struct gve_priv *priv) 249 { 250 set_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 251 } 252 253 static inline void gve_clear_reset_in_progress(struct gve_priv *priv) 254 { 255 clear_bit(GVE_PRIV_FLAGS_RESET_IN_PROGRESS, &priv->service_task_flags); 256 } 257 258 static inline bool gve_get_probe_in_progress(struct gve_priv *priv) 259 { 260 return test_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, 261 &priv->service_task_flags); 262 } 263 264 static inline void gve_set_probe_in_progress(struct gve_priv *priv) 265 { 266 set_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 267 } 268 269 static inline void gve_clear_probe_in_progress(struct gve_priv *priv) 270 { 271 clear_bit(GVE_PRIV_FLAGS_PROBE_IN_PROGRESS, &priv->service_task_flags); 272 } 273 274 static inline bool gve_get_admin_queue_ok(struct gve_priv *priv) 275 { 276 return test_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 277 } 278 279 static inline void gve_set_admin_queue_ok(struct gve_priv *priv) 280 { 281 set_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 282 } 283 284 static inline void gve_clear_admin_queue_ok(struct gve_priv *priv) 285 { 286 clear_bit(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK, &priv->state_flags); 287 } 288 289 static inline bool gve_get_device_resources_ok(struct gve_priv *priv) 290 { 291 return test_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 292 } 293 294 static inline void gve_set_device_resources_ok(struct gve_priv *priv) 295 { 296 set_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 297 } 298 299 static inline void gve_clear_device_resources_ok(struct gve_priv *priv) 300 { 301 clear_bit(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK, &priv->state_flags); 302 } 303 304 static inline bool gve_get_device_rings_ok(struct gve_priv *priv) 305 { 306 return test_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 307 } 308 309 static inline void gve_set_device_rings_ok(struct gve_priv *priv) 310 { 311 set_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 312 } 313 314 static inline void gve_clear_device_rings_ok(struct gve_priv *priv) 315 { 316 clear_bit(GVE_PRIV_FLAGS_DEVICE_RINGS_OK, &priv->state_flags); 317 } 318 319 static inline bool gve_get_napi_enabled(struct gve_priv *priv) 320 { 321 return test_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 322 } 323 324 static inline void gve_set_napi_enabled(struct gve_priv *priv) 325 { 326 set_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 327 } 328 329 static inline void gve_clear_napi_enabled(struct gve_priv *priv) 330 { 331 clear_bit(GVE_PRIV_FLAGS_NAPI_ENABLED, &priv->state_flags); 332 } 333 334 /* Returns the address of the ntfy_blocks irq doorbell 335 */ 336 static inline __be32 __iomem *gve_irq_doorbell(struct gve_priv *priv, 337 struct gve_notify_block *block) 338 { 339 return &priv->db_bar2[be32_to_cpu(block->irq_db_index)]; 340 } 341 342 /* Returns the index into ntfy_blocks of the given tx ring's block 343 */ 344 static inline u32 gve_tx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 345 { 346 return queue_idx; 347 } 348 349 /* Returns the index into ntfy_blocks of the given rx ring's block 350 */ 351 static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx) 352 { 353 return (priv->num_ntfy_blks / 2) + queue_idx; 354 } 355 356 /* Returns the number of tx queue page lists 357 */ 358 static inline u32 gve_num_tx_qpls(struct gve_priv *priv) 359 { 360 return priv->tx_cfg.num_queues; 361 } 362 363 /* Returns the number of rx queue page lists 364 */ 365 static inline u32 gve_num_rx_qpls(struct gve_priv *priv) 366 { 367 return priv->rx_cfg.num_queues; 368 } 369 370 /* Returns a pointer to the next available tx qpl in the list of qpls 371 */ 372 static inline 373 struct gve_queue_page_list *gve_assign_tx_qpl(struct gve_priv *priv) 374 { 375 int id = find_first_zero_bit(priv->qpl_cfg.qpl_id_map, 376 priv->qpl_cfg.qpl_map_size); 377 378 /* we are out of tx qpls */ 379 if (id >= gve_num_tx_qpls(priv)) 380 return NULL; 381 382 set_bit(id, priv->qpl_cfg.qpl_id_map); 383 return &priv->qpls[id]; 384 } 385 386 /* Returns a pointer to the next available rx qpl in the list of qpls 387 */ 388 static inline 389 struct gve_queue_page_list *gve_assign_rx_qpl(struct gve_priv *priv) 390 { 391 int id = find_next_zero_bit(priv->qpl_cfg.qpl_id_map, 392 priv->qpl_cfg.qpl_map_size, 393 gve_num_tx_qpls(priv)); 394 395 /* we are out of rx qpls */ 396 if (id == priv->qpl_cfg.qpl_map_size) 397 return NULL; 398 399 set_bit(id, priv->qpl_cfg.qpl_id_map); 400 return &priv->qpls[id]; 401 } 402 403 /* Unassigns the qpl with the given id 404 */ 405 static inline void gve_unassign_qpl(struct gve_priv *priv, int id) 406 { 407 clear_bit(id, priv->qpl_cfg.qpl_id_map); 408 } 409 410 /* Returns the correct dma direction for tx and rx qpls 411 */ 412 static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, 413 int id) 414 { 415 if (id < gve_num_tx_qpls(priv)) 416 return DMA_TO_DEVICE; 417 else 418 return DMA_FROM_DEVICE; 419 } 420 421 /* Returns true if the max mtu allows page recycling */ 422 static inline bool gve_can_recycle_pages(struct net_device *dev) 423 { 424 /* We can't recycle the pages if we can't fit a packet into half a 425 * page. 426 */ 427 return dev->max_mtu <= PAGE_SIZE / 2; 428 } 429 430 /* buffers */ 431 int gve_alloc_page(struct device *dev, struct page **page, dma_addr_t *dma, 432 enum dma_data_direction); 433 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 434 enum dma_data_direction); 435 /* tx handling */ 436 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev); 437 bool gve_tx_poll(struct gve_notify_block *block, int budget); 438 int gve_tx_alloc_rings(struct gve_priv *priv); 439 void gve_tx_free_rings(struct gve_priv *priv); 440 __be32 gve_tx_load_event_counter(struct gve_priv *priv, 441 struct gve_tx_ring *tx); 442 /* rx handling */ 443 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx); 444 bool gve_rx_poll(struct gve_notify_block *block, int budget); 445 int gve_rx_alloc_rings(struct gve_priv *priv); 446 void gve_rx_free_rings(struct gve_priv *priv); 447 bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget, 448 netdev_features_t feat); 449 /* Reset */ 450 void gve_schedule_reset(struct gve_priv *priv); 451 int gve_reset(struct gve_priv *priv, bool attempt_teardown); 452 int gve_adjust_queues(struct gve_priv *priv, 453 struct gve_queue_config new_rx_config, 454 struct gve_queue_config new_tx_config); 455 /* exported by ethtool.c */ 456 extern const struct ethtool_ops gve_ethtool_ops; 457 /* needed by ethtool */ 458 extern const char gve_version_str[]; 459 #endif /* _GVE_H_ */ 460