1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2021 Google, Inc.
5  */
6 
7 #include "gve.h"
8 #include "gve_adminq.h"
9 #include "gve_utils.h"
10 #include "gve_dqo.h"
11 #include <net/ip.h>
12 #include <linux/tcp.h>
13 #include <linux/slab.h>
14 #include <linux/skbuff.h>
15 
16 /* Returns true if tx_bufs are available. */
17 static bool gve_has_free_tx_qpl_bufs(struct gve_tx_ring *tx, int count)
18 {
19 	int num_avail;
20 
21 	if (!tx->dqo.qpl)
22 		return true;
23 
24 	num_avail = tx->dqo.num_tx_qpl_bufs -
25 		(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
26 		 tx->dqo_tx.free_tx_qpl_buf_cnt);
27 
28 	if (count <= num_avail)
29 		return true;
30 
31 	/* Update cached value from dqo_compl. */
32 	tx->dqo_tx.free_tx_qpl_buf_cnt =
33 		atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_cnt);
34 
35 	num_avail = tx->dqo.num_tx_qpl_bufs -
36 		(tx->dqo_tx.alloc_tx_qpl_buf_cnt -
37 		 tx->dqo_tx.free_tx_qpl_buf_cnt);
38 
39 	return count <= num_avail;
40 }
41 
42 static s16
43 gve_alloc_tx_qpl_buf(struct gve_tx_ring *tx)
44 {
45 	s16 index;
46 
47 	index = tx->dqo_tx.free_tx_qpl_buf_head;
48 
49 	/* No TX buffers available, try to steal the list from the
50 	 * completion handler.
51 	 */
52 	if (unlikely(index == -1)) {
53 		tx->dqo_tx.free_tx_qpl_buf_head =
54 			atomic_xchg(&tx->dqo_compl.free_tx_qpl_buf_head, -1);
55 		index = tx->dqo_tx.free_tx_qpl_buf_head;
56 
57 		if (unlikely(index == -1))
58 			return index;
59 	}
60 
61 	/* Remove TX buf from free list */
62 	tx->dqo_tx.free_tx_qpl_buf_head = tx->dqo.tx_qpl_buf_next[index];
63 
64 	return index;
65 }
66 
67 static void
68 gve_free_tx_qpl_bufs(struct gve_tx_ring *tx,
69 		     struct gve_tx_pending_packet_dqo *pkt)
70 {
71 	s16 index;
72 	int i;
73 
74 	if (!pkt->num_bufs)
75 		return;
76 
77 	index = pkt->tx_qpl_buf_ids[0];
78 	/* Create a linked list of buffers to be added to the free list */
79 	for (i = 1; i < pkt->num_bufs; i++) {
80 		tx->dqo.tx_qpl_buf_next[index] = pkt->tx_qpl_buf_ids[i];
81 		index = pkt->tx_qpl_buf_ids[i];
82 	}
83 
84 	while (true) {
85 		s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_head);
86 
87 		tx->dqo.tx_qpl_buf_next[index] = old_head;
88 		if (atomic_cmpxchg(&tx->dqo_compl.free_tx_qpl_buf_head,
89 				   old_head,
90 				   pkt->tx_qpl_buf_ids[0]) == old_head) {
91 			break;
92 		}
93 	}
94 
95 	atomic_add(pkt->num_bufs, &tx->dqo_compl.free_tx_qpl_buf_cnt);
96 	pkt->num_bufs = 0;
97 }
98 
99 /* Returns true if a gve_tx_pending_packet_dqo object is available. */
100 static bool gve_has_pending_packet(struct gve_tx_ring *tx)
101 {
102 	/* Check TX path's list. */
103 	if (tx->dqo_tx.free_pending_packets != -1)
104 		return true;
105 
106 	/* Check completion handler's list. */
107 	if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1)
108 		return true;
109 
110 	return false;
111 }
112 
113 static struct gve_tx_pending_packet_dqo *
114 gve_alloc_pending_packet(struct gve_tx_ring *tx)
115 {
116 	struct gve_tx_pending_packet_dqo *pending_packet;
117 	s16 index;
118 
119 	index = tx->dqo_tx.free_pending_packets;
120 
121 	/* No pending_packets available, try to steal the list from the
122 	 * completion handler.
123 	 */
124 	if (unlikely(index == -1)) {
125 		tx->dqo_tx.free_pending_packets =
126 			atomic_xchg(&tx->dqo_compl.free_pending_packets, -1);
127 		index = tx->dqo_tx.free_pending_packets;
128 
129 		if (unlikely(index == -1))
130 			return NULL;
131 	}
132 
133 	pending_packet = &tx->dqo.pending_packets[index];
134 
135 	/* Remove pending_packet from free list */
136 	tx->dqo_tx.free_pending_packets = pending_packet->next;
137 	pending_packet->state = GVE_PACKET_STATE_PENDING_DATA_COMPL;
138 
139 	return pending_packet;
140 }
141 
142 static void
143 gve_free_pending_packet(struct gve_tx_ring *tx,
144 			struct gve_tx_pending_packet_dqo *pending_packet)
145 {
146 	s16 index = pending_packet - tx->dqo.pending_packets;
147 
148 	pending_packet->state = GVE_PACKET_STATE_UNALLOCATED;
149 	while (true) {
150 		s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets);
151 
152 		pending_packet->next = old_head;
153 		if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets,
154 				   old_head, index) == old_head) {
155 			break;
156 		}
157 	}
158 }
159 
160 /* gve_tx_free_desc - Cleans up all pending tx requests and buffers.
161  */
162 static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx)
163 {
164 	int i;
165 
166 	for (i = 0; i < tx->dqo.num_pending_packets; i++) {
167 		struct gve_tx_pending_packet_dqo *cur_state =
168 			&tx->dqo.pending_packets[i];
169 		int j;
170 
171 		for (j = 0; j < cur_state->num_bufs; j++) {
172 			if (j == 0) {
173 				dma_unmap_single(tx->dev,
174 					dma_unmap_addr(cur_state, dma[j]),
175 					dma_unmap_len(cur_state, len[j]),
176 					DMA_TO_DEVICE);
177 			} else {
178 				dma_unmap_page(tx->dev,
179 					dma_unmap_addr(cur_state, dma[j]),
180 					dma_unmap_len(cur_state, len[j]),
181 					DMA_TO_DEVICE);
182 			}
183 		}
184 		if (cur_state->skb) {
185 			dev_consume_skb_any(cur_state->skb);
186 			cur_state->skb = NULL;
187 		}
188 	}
189 }
190 
191 static void gve_tx_free_ring_dqo(struct gve_priv *priv, int idx)
192 {
193 	struct gve_tx_ring *tx = &priv->tx[idx];
194 	struct device *hdev = &priv->pdev->dev;
195 	size_t bytes;
196 
197 	gve_tx_remove_from_block(priv, idx);
198 
199 	if (tx->q_resources) {
200 		dma_free_coherent(hdev, sizeof(*tx->q_resources),
201 				  tx->q_resources, tx->q_resources_bus);
202 		tx->q_resources = NULL;
203 	}
204 
205 	if (tx->dqo.compl_ring) {
206 		bytes = sizeof(tx->dqo.compl_ring[0]) *
207 			(tx->dqo.complq_mask + 1);
208 		dma_free_coherent(hdev, bytes, tx->dqo.compl_ring,
209 				  tx->complq_bus_dqo);
210 		tx->dqo.compl_ring = NULL;
211 	}
212 
213 	if (tx->dqo.tx_ring) {
214 		bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1);
215 		dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus);
216 		tx->dqo.tx_ring = NULL;
217 	}
218 
219 	kvfree(tx->dqo.pending_packets);
220 	tx->dqo.pending_packets = NULL;
221 
222 	kvfree(tx->dqo.tx_qpl_buf_next);
223 	tx->dqo.tx_qpl_buf_next = NULL;
224 
225 	if (tx->dqo.qpl) {
226 		gve_unassign_qpl(priv, tx->dqo.qpl->id);
227 		tx->dqo.qpl = NULL;
228 	}
229 
230 	netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx);
231 }
232 
233 static int gve_tx_qpl_buf_init(struct gve_tx_ring *tx)
234 {
235 	int num_tx_qpl_bufs = GVE_TX_BUFS_PER_PAGE_DQO *
236 		tx->dqo.qpl->num_entries;
237 	int i;
238 
239 	tx->dqo.tx_qpl_buf_next = kvcalloc(num_tx_qpl_bufs,
240 					   sizeof(tx->dqo.tx_qpl_buf_next[0]),
241 					   GFP_KERNEL);
242 	if (!tx->dqo.tx_qpl_buf_next)
243 		return -ENOMEM;
244 
245 	tx->dqo.num_tx_qpl_bufs = num_tx_qpl_bufs;
246 
247 	/* Generate free TX buf list */
248 	for (i = 0; i < num_tx_qpl_bufs - 1; i++)
249 		tx->dqo.tx_qpl_buf_next[i] = i + 1;
250 	tx->dqo.tx_qpl_buf_next[num_tx_qpl_bufs - 1] = -1;
251 
252 	atomic_set_release(&tx->dqo_compl.free_tx_qpl_buf_head, -1);
253 	return 0;
254 }
255 
256 static int gve_tx_alloc_ring_dqo(struct gve_priv *priv, int idx)
257 {
258 	struct gve_tx_ring *tx = &priv->tx[idx];
259 	struct device *hdev = &priv->pdev->dev;
260 	int num_pending_packets;
261 	size_t bytes;
262 	int i;
263 
264 	memset(tx, 0, sizeof(*tx));
265 	tx->q_num = idx;
266 	tx->dev = &priv->pdev->dev;
267 	tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx);
268 	atomic_set_release(&tx->dqo_compl.hw_tx_head, 0);
269 
270 	/* Queue sizes must be a power of 2 */
271 	tx->mask = priv->tx_desc_cnt - 1;
272 	tx->dqo.complq_mask = priv->queue_format == GVE_DQO_RDA_FORMAT ?
273 		priv->options_dqo_rda.tx_comp_ring_entries - 1 :
274 		tx->mask;
275 
276 	/* The max number of pending packets determines the maximum number of
277 	 * descriptors which maybe written to the completion queue.
278 	 *
279 	 * We must set the number small enough to make sure we never overrun the
280 	 * completion queue.
281 	 */
282 	num_pending_packets = tx->dqo.complq_mask + 1;
283 
284 	/* Reserve space for descriptor completions, which will be reported at
285 	 * most every GVE_TX_MIN_RE_INTERVAL packets.
286 	 */
287 	num_pending_packets -=
288 		(tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL;
289 
290 	/* Each packet may have at most 2 buffer completions if it receives both
291 	 * a miss and reinjection completion.
292 	 */
293 	num_pending_packets /= 2;
294 
295 	tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX);
296 	tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets,
297 					   sizeof(tx->dqo.pending_packets[0]),
298 					   GFP_KERNEL);
299 	if (!tx->dqo.pending_packets)
300 		goto err;
301 
302 	/* Set up linked list of pending packets */
303 	for (i = 0; i < tx->dqo.num_pending_packets - 1; i++)
304 		tx->dqo.pending_packets[i].next = i + 1;
305 
306 	tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1;
307 	atomic_set_release(&tx->dqo_compl.free_pending_packets, -1);
308 	tx->dqo_compl.miss_completions.head = -1;
309 	tx->dqo_compl.miss_completions.tail = -1;
310 	tx->dqo_compl.timed_out_completions.head = -1;
311 	tx->dqo_compl.timed_out_completions.tail = -1;
312 
313 	bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1);
314 	tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL);
315 	if (!tx->dqo.tx_ring)
316 		goto err;
317 
318 	bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1);
319 	tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes,
320 						&tx->complq_bus_dqo,
321 						GFP_KERNEL);
322 	if (!tx->dqo.compl_ring)
323 		goto err;
324 
325 	tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources),
326 					     &tx->q_resources_bus, GFP_KERNEL);
327 	if (!tx->q_resources)
328 		goto err;
329 
330 	if (gve_is_qpl(priv)) {
331 		tx->dqo.qpl = gve_assign_tx_qpl(priv, idx);
332 		if (!tx->dqo.qpl)
333 			goto err;
334 
335 		if (gve_tx_qpl_buf_init(tx))
336 			goto err;
337 	}
338 
339 	gve_tx_add_to_block(priv, idx);
340 
341 	return 0;
342 
343 err:
344 	gve_tx_free_ring_dqo(priv, idx);
345 	return -ENOMEM;
346 }
347 
348 int gve_tx_alloc_rings_dqo(struct gve_priv *priv)
349 {
350 	int err = 0;
351 	int i;
352 
353 	for (i = 0; i < priv->tx_cfg.num_queues; i++) {
354 		err = gve_tx_alloc_ring_dqo(priv, i);
355 		if (err) {
356 			netif_err(priv, drv, priv->dev,
357 				  "Failed to alloc tx ring=%d: err=%d\n",
358 				  i, err);
359 			goto err;
360 		}
361 	}
362 
363 	return 0;
364 
365 err:
366 	for (i--; i >= 0; i--)
367 		gve_tx_free_ring_dqo(priv, i);
368 
369 	return err;
370 }
371 
372 void gve_tx_free_rings_dqo(struct gve_priv *priv)
373 {
374 	int i;
375 
376 	for (i = 0; i < priv->tx_cfg.num_queues; i++) {
377 		struct gve_tx_ring *tx = &priv->tx[i];
378 
379 		gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL);
380 		netdev_tx_reset_queue(tx->netdev_txq);
381 		gve_tx_clean_pending_packets(tx);
382 
383 		gve_tx_free_ring_dqo(priv, i);
384 	}
385 }
386 
387 /* Returns the number of slots available in the ring */
388 static u32 num_avail_tx_slots(const struct gve_tx_ring *tx)
389 {
390 	u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask;
391 
392 	return tx->mask - num_used;
393 }
394 
395 static bool gve_has_avail_slots_tx_dqo(struct gve_tx_ring *tx,
396 				       int desc_count, int buf_count)
397 {
398 	return gve_has_pending_packet(tx) &&
399 		   num_avail_tx_slots(tx) >= desc_count &&
400 		   gve_has_free_tx_qpl_bufs(tx, buf_count);
401 }
402 
403 /* Stops the queue if available descriptors is less than 'count'.
404  * Return: 0 if stop is not required.
405  */
406 static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx,
407 				 int desc_count, int buf_count)
408 {
409 	if (likely(gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count)))
410 		return 0;
411 
412 	/* Update cached TX head pointer */
413 	tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head);
414 
415 	if (likely(gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count)))
416 		return 0;
417 
418 	/* No space, so stop the queue */
419 	tx->stop_queue++;
420 	netif_tx_stop_queue(tx->netdev_txq);
421 
422 	/* Sync with restarting queue in `gve_tx_poll_dqo()` */
423 	mb();
424 
425 	/* After stopping queue, check if we can transmit again in order to
426 	 * avoid TOCTOU bug.
427 	 */
428 	tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head);
429 
430 	if (likely(!gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count)))
431 		return -EBUSY;
432 
433 	netif_tx_start_queue(tx->netdev_txq);
434 	tx->wake_queue++;
435 	return 0;
436 }
437 
438 static void gve_extract_tx_metadata_dqo(const struct sk_buff *skb,
439 					struct gve_tx_metadata_dqo *metadata)
440 {
441 	memset(metadata, 0, sizeof(*metadata));
442 	metadata->version = GVE_TX_METADATA_VERSION_DQO;
443 
444 	if (skb->l4_hash) {
445 		u16 path_hash = skb->hash ^ (skb->hash >> 16);
446 
447 		path_hash &= (1 << 15) - 1;
448 		if (unlikely(path_hash == 0))
449 			path_hash = ~path_hash;
450 
451 		metadata->path_hash = path_hash;
452 	}
453 }
454 
455 static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx,
456 				     struct sk_buff *skb, u32 len, u64 addr,
457 				     s16 compl_tag, bool eop, bool is_gso)
458 {
459 	const bool checksum_offload_en = skb->ip_summed == CHECKSUM_PARTIAL;
460 
461 	while (len > 0) {
462 		struct gve_tx_pkt_desc_dqo *desc =
463 			&tx->dqo.tx_ring[*desc_idx].pkt;
464 		u32 cur_len = min_t(u32, len, GVE_TX_MAX_BUF_SIZE_DQO);
465 		bool cur_eop = eop && cur_len == len;
466 
467 		*desc = (struct gve_tx_pkt_desc_dqo){
468 			.buf_addr = cpu_to_le64(addr),
469 			.dtype = GVE_TX_PKT_DESC_DTYPE_DQO,
470 			.end_of_packet = cur_eop,
471 			.checksum_offload_enable = checksum_offload_en,
472 			.compl_tag = cpu_to_le16(compl_tag),
473 			.buf_size = cur_len,
474 		};
475 
476 		addr += cur_len;
477 		len -= cur_len;
478 		*desc_idx = (*desc_idx + 1) & tx->mask;
479 	}
480 }
481 
482 /* Validates and prepares `skb` for TSO.
483  *
484  * Returns header length, or < 0 if invalid.
485  */
486 static int gve_prep_tso(struct sk_buff *skb)
487 {
488 	struct tcphdr *tcp;
489 	int header_len;
490 	u32 paylen;
491 	int err;
492 
493 	/* Note: HW requires MSS (gso_size) to be <= 9728 and the total length
494 	 * of the TSO to be <= 262143.
495 	 *
496 	 * However, we don't validate these because:
497 	 * - Hypervisor enforces a limit of 9K MTU
498 	 * - Kernel will not produce a TSO larger than 64k
499 	 */
500 
501 	if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO))
502 		return -1;
503 
504 	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)))
505 		return -EINVAL;
506 
507 	/* Needed because we will modify header. */
508 	err = skb_cow_head(skb, 0);
509 	if (err < 0)
510 		return err;
511 
512 	tcp = tcp_hdr(skb);
513 	paylen = skb->len - skb_transport_offset(skb);
514 	csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen));
515 	header_len = skb_tcp_all_headers(skb);
516 
517 	if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO))
518 		return -EINVAL;
519 
520 	return header_len;
521 }
522 
523 static void gve_tx_fill_tso_ctx_desc(struct gve_tx_tso_context_desc_dqo *desc,
524 				     const struct sk_buff *skb,
525 				     const struct gve_tx_metadata_dqo *metadata,
526 				     int header_len)
527 {
528 	*desc = (struct gve_tx_tso_context_desc_dqo){
529 		.header_len = header_len,
530 		.cmd_dtype = {
531 			.dtype = GVE_TX_TSO_CTX_DESC_DTYPE_DQO,
532 			.tso = 1,
533 		},
534 		.flex0 = metadata->bytes[0],
535 		.flex5 = metadata->bytes[5],
536 		.flex6 = metadata->bytes[6],
537 		.flex7 = metadata->bytes[7],
538 		.flex8 = metadata->bytes[8],
539 		.flex9 = metadata->bytes[9],
540 		.flex10 = metadata->bytes[10],
541 		.flex11 = metadata->bytes[11],
542 	};
543 	desc->tso_total_len = skb->len - header_len;
544 	desc->mss = skb_shinfo(skb)->gso_size;
545 }
546 
547 static void
548 gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc,
549 			     const struct gve_tx_metadata_dqo *metadata)
550 {
551 	*desc = (struct gve_tx_general_context_desc_dqo){
552 		.flex0 = metadata->bytes[0],
553 		.flex1 = metadata->bytes[1],
554 		.flex2 = metadata->bytes[2],
555 		.flex3 = metadata->bytes[3],
556 		.flex4 = metadata->bytes[4],
557 		.flex5 = metadata->bytes[5],
558 		.flex6 = metadata->bytes[6],
559 		.flex7 = metadata->bytes[7],
560 		.flex8 = metadata->bytes[8],
561 		.flex9 = metadata->bytes[9],
562 		.flex10 = metadata->bytes[10],
563 		.flex11 = metadata->bytes[11],
564 		.cmd_dtype = {.dtype = GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO},
565 	};
566 }
567 
568 static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
569 				      struct sk_buff *skb,
570 				      struct gve_tx_pending_packet_dqo *pkt,
571 				      s16 completion_tag,
572 				      u32 *desc_idx,
573 				      bool is_gso)
574 {
575 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
576 	int i;
577 
578 	/* Note: HW requires that the size of a non-TSO packet be within the
579 	 * range of [17, 9728].
580 	 *
581 	 * We don't double check because
582 	 * - We limited `netdev->min_mtu` to ETH_MIN_MTU.
583 	 * - Hypervisor won't allow MTU larger than 9216.
584 	 */
585 
586 	pkt->num_bufs = 0;
587 	/* Map the linear portion of skb */
588 	{
589 		u32 len = skb_headlen(skb);
590 		dma_addr_t addr;
591 
592 		addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
593 		if (unlikely(dma_mapping_error(tx->dev, addr)))
594 			goto err;
595 
596 		dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
597 		dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
598 		++pkt->num_bufs;
599 
600 		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, len, addr,
601 					 completion_tag,
602 					 /*eop=*/shinfo->nr_frags == 0, is_gso);
603 	}
604 
605 	for (i = 0; i < shinfo->nr_frags; i++) {
606 		const skb_frag_t *frag = &shinfo->frags[i];
607 		bool is_eop = i == (shinfo->nr_frags - 1);
608 		u32 len = skb_frag_size(frag);
609 		dma_addr_t addr;
610 
611 		addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE);
612 		if (unlikely(dma_mapping_error(tx->dev, addr)))
613 			goto err;
614 
615 		dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
616 		dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
617 		++pkt->num_bufs;
618 
619 		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, len, addr,
620 					 completion_tag, is_eop, is_gso);
621 	}
622 
623 	return 0;
624 err:
625 	for (i = 0; i < pkt->num_bufs; i++) {
626 		if (i == 0) {
627 			dma_unmap_single(tx->dev,
628 					 dma_unmap_addr(pkt, dma[i]),
629 					 dma_unmap_len(pkt, len[i]),
630 					 DMA_TO_DEVICE);
631 		} else {
632 			dma_unmap_page(tx->dev,
633 				       dma_unmap_addr(pkt, dma[i]),
634 				       dma_unmap_len(pkt, len[i]),
635 				       DMA_TO_DEVICE);
636 		}
637 	}
638 	pkt->num_bufs = 0;
639 	return -1;
640 }
641 
642 /* Tx buffer i corresponds to
643  * qpl_page_id = i / GVE_TX_BUFS_PER_PAGE_DQO
644  * qpl_page_offset = (i % GVE_TX_BUFS_PER_PAGE_DQO) * GVE_TX_BUF_SIZE_DQO
645  */
646 static void gve_tx_buf_get_addr(struct gve_tx_ring *tx,
647 				s16 index,
648 				void **va, dma_addr_t *dma_addr)
649 {
650 	int page_id = index >> (PAGE_SHIFT - GVE_TX_BUF_SHIFT_DQO);
651 	int offset = (index & (GVE_TX_BUFS_PER_PAGE_DQO - 1)) << GVE_TX_BUF_SHIFT_DQO;
652 
653 	*va = page_address(tx->dqo.qpl->pages[page_id]) + offset;
654 	*dma_addr = tx->dqo.qpl->page_buses[page_id] + offset;
655 }
656 
657 static int gve_tx_add_skb_copy_dqo(struct gve_tx_ring *tx,
658 				   struct sk_buff *skb,
659 				   struct gve_tx_pending_packet_dqo *pkt,
660 				   s16 completion_tag,
661 				   u32 *desc_idx,
662 				   bool is_gso)
663 {
664 	u32 copy_offset = 0;
665 	dma_addr_t dma_addr;
666 	u32 copy_len;
667 	s16 index;
668 	void *va;
669 
670 	/* Break the packet into buffer size chunks */
671 	pkt->num_bufs = 0;
672 	while (copy_offset < skb->len) {
673 		index = gve_alloc_tx_qpl_buf(tx);
674 		if (unlikely(index == -1))
675 			goto err;
676 
677 		gve_tx_buf_get_addr(tx, index, &va, &dma_addr);
678 		copy_len = min_t(u32, GVE_TX_BUF_SIZE_DQO,
679 				 skb->len - copy_offset);
680 		skb_copy_bits(skb, copy_offset, va, copy_len);
681 
682 		copy_offset += copy_len;
683 		dma_sync_single_for_device(tx->dev, dma_addr,
684 					   copy_len, DMA_TO_DEVICE);
685 		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb,
686 					 copy_len,
687 					 dma_addr,
688 					 completion_tag,
689 					 copy_offset == skb->len,
690 					 is_gso);
691 
692 		pkt->tx_qpl_buf_ids[pkt->num_bufs] = index;
693 		++tx->dqo_tx.alloc_tx_qpl_buf_cnt;
694 		++pkt->num_bufs;
695 	}
696 
697 	return 0;
698 err:
699 	/* Should not be here if gve_has_free_tx_qpl_bufs() check is correct */
700 	gve_free_tx_qpl_bufs(tx, pkt);
701 	return -ENOMEM;
702 }
703 
704 /* Returns 0 on success, or < 0 on error.
705  *
706  * Before this function is called, the caller must ensure
707  * gve_has_pending_packet(tx) returns true.
708  */
709 static int gve_tx_add_skb_dqo(struct gve_tx_ring *tx,
710 			      struct sk_buff *skb)
711 {
712 	const bool is_gso = skb_is_gso(skb);
713 	u32 desc_idx = tx->dqo_tx.tail;
714 	struct gve_tx_pending_packet_dqo *pkt;
715 	struct gve_tx_metadata_dqo metadata;
716 	s16 completion_tag;
717 
718 	pkt = gve_alloc_pending_packet(tx);
719 	pkt->skb = skb;
720 	completion_tag = pkt - tx->dqo.pending_packets;
721 
722 	gve_extract_tx_metadata_dqo(skb, &metadata);
723 	if (is_gso) {
724 		int header_len = gve_prep_tso(skb);
725 
726 		if (unlikely(header_len < 0))
727 			goto err;
728 
729 		gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx,
730 					 skb, &metadata, header_len);
731 		desc_idx = (desc_idx + 1) & tx->mask;
732 	}
733 
734 	gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx,
735 				     &metadata);
736 	desc_idx = (desc_idx + 1) & tx->mask;
737 
738 	if (tx->dqo.qpl) {
739 		if (gve_tx_add_skb_copy_dqo(tx, skb, pkt,
740 					    completion_tag,
741 					    &desc_idx, is_gso))
742 			goto err;
743 	}  else {
744 		if (gve_tx_add_skb_no_copy_dqo(tx, skb, pkt,
745 					       completion_tag,
746 					       &desc_idx, is_gso))
747 			goto err;
748 	}
749 
750 	tx->dqo_tx.posted_packet_desc_cnt += pkt->num_bufs;
751 
752 	/* Commit the changes to our state */
753 	tx->dqo_tx.tail = desc_idx;
754 
755 	/* Request a descriptor completion on the last descriptor of the
756 	 * packet if we are allowed to by the HW enforced interval.
757 	 */
758 	{
759 		u32 last_desc_idx = (desc_idx - 1) & tx->mask;
760 		u32 last_report_event_interval =
761 			(last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask;
762 
763 		if (unlikely(last_report_event_interval >=
764 			     GVE_TX_MIN_RE_INTERVAL)) {
765 			tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true;
766 			tx->dqo_tx.last_re_idx = last_desc_idx;
767 		}
768 	}
769 
770 	return 0;
771 
772 err:
773 	pkt->skb = NULL;
774 	gve_free_pending_packet(tx, pkt);
775 
776 	return -1;
777 }
778 
779 static int gve_num_descs_per_buf(size_t size)
780 {
781 	return DIV_ROUND_UP(size, GVE_TX_MAX_BUF_SIZE_DQO);
782 }
783 
784 static int gve_num_buffer_descs_needed(const struct sk_buff *skb)
785 {
786 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
787 	int num_descs;
788 	int i;
789 
790 	num_descs = gve_num_descs_per_buf(skb_headlen(skb));
791 
792 	for (i = 0; i < shinfo->nr_frags; i++) {
793 		unsigned int frag_size = skb_frag_size(&shinfo->frags[i]);
794 
795 		num_descs += gve_num_descs_per_buf(frag_size);
796 	}
797 
798 	return num_descs;
799 }
800 
801 /* Returns true if HW is capable of sending TSO represented by `skb`.
802  *
803  * Each segment must not span more than GVE_TX_MAX_DATA_DESCS buffers.
804  * - The header is counted as one buffer for every single segment.
805  * - A buffer which is split between two segments is counted for both.
806  * - If a buffer contains both header and payload, it is counted as two buffers.
807  */
808 static bool gve_can_send_tso(const struct sk_buff *skb)
809 {
810 	const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1;
811 	const struct skb_shared_info *shinfo = skb_shinfo(skb);
812 	const int header_len = skb_tcp_all_headers(skb);
813 	const int gso_size = shinfo->gso_size;
814 	int cur_seg_num_bufs;
815 	int cur_seg_size;
816 	int i;
817 
818 	cur_seg_size = skb_headlen(skb) - header_len;
819 	cur_seg_num_bufs = cur_seg_size > 0;
820 
821 	for (i = 0; i < shinfo->nr_frags; i++) {
822 		if (cur_seg_size >= gso_size) {
823 			cur_seg_size %= gso_size;
824 			cur_seg_num_bufs = cur_seg_size > 0;
825 		}
826 
827 		if (unlikely(++cur_seg_num_bufs > max_bufs_per_seg))
828 			return false;
829 
830 		cur_seg_size += skb_frag_size(&shinfo->frags[i]);
831 	}
832 
833 	return true;
834 }
835 
836 /* Attempt to transmit specified SKB.
837  *
838  * Returns 0 if the SKB was transmitted or dropped.
839  * Returns -1 if there is not currently enough space to transmit the SKB.
840  */
841 static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx,
842 			  struct sk_buff *skb)
843 {
844 	int num_buffer_descs;
845 	int total_num_descs;
846 
847 	if (tx->dqo.qpl) {
848 		if (skb_is_gso(skb))
849 			if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
850 				goto drop;
851 
852 		/* We do not need to verify the number of buffers used per
853 		 * packet or per segment in case of TSO as with 2K size buffers
854 		 * none of the TX packet rules would be violated.
855 		 *
856 		 * gve_can_send_tso() checks that each TCP segment of gso_size is
857 		 * not distributed over more than 9 SKB frags..
858 		 */
859 		num_buffer_descs = DIV_ROUND_UP(skb->len, GVE_TX_BUF_SIZE_DQO);
860 	} else {
861 		if (skb_is_gso(skb)) {
862 			/* If TSO doesn't meet HW requirements, attempt to linearize the
863 			 * packet.
864 			 */
865 			if (unlikely(!gve_can_send_tso(skb) &&
866 				     skb_linearize(skb) < 0)) {
867 				net_err_ratelimited("%s: Failed to transmit TSO packet\n",
868 						    priv->dev->name);
869 				goto drop;
870 			}
871 
872 			if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
873 				goto drop;
874 
875 			num_buffer_descs = gve_num_buffer_descs_needed(skb);
876 		} else {
877 			num_buffer_descs = gve_num_buffer_descs_needed(skb);
878 
879 			if (unlikely(num_buffer_descs > GVE_TX_MAX_DATA_DESCS)) {
880 				if (unlikely(skb_linearize(skb) < 0))
881 					goto drop;
882 
883 				num_buffer_descs = 1;
884 			}
885 		}
886 	}
887 
888 	/* Metadata + (optional TSO) + data descriptors. */
889 	total_num_descs = 1 + skb_is_gso(skb) + num_buffer_descs;
890 	if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs +
891 			GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP,
892 			num_buffer_descs))) {
893 		return -1;
894 	}
895 
896 	if (unlikely(gve_tx_add_skb_dqo(tx, skb) < 0))
897 		goto drop;
898 
899 	netdev_tx_sent_queue(tx->netdev_txq, skb->len);
900 	skb_tx_timestamp(skb);
901 	return 0;
902 
903 drop:
904 	tx->dropped_pkt++;
905 	dev_kfree_skb_any(skb);
906 	return 0;
907 }
908 
909 /* Transmit a given skb and ring the doorbell. */
910 netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev)
911 {
912 	struct gve_priv *priv = netdev_priv(dev);
913 	struct gve_tx_ring *tx;
914 
915 	tx = &priv->tx[skb_get_queue_mapping(skb)];
916 	if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) {
917 		/* We need to ring the txq doorbell -- we have stopped the Tx
918 		 * queue for want of resources, but prior calls to gve_tx()
919 		 * may have added descriptors without ringing the doorbell.
920 		 */
921 		gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
922 		return NETDEV_TX_BUSY;
923 	}
924 
925 	if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more())
926 		return NETDEV_TX_OK;
927 
928 	gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail);
929 	return NETDEV_TX_OK;
930 }
931 
932 static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list,
933 			struct gve_tx_pending_packet_dqo *pending_packet)
934 {
935 	s16 old_tail, index;
936 
937 	index = pending_packet - tx->dqo.pending_packets;
938 	old_tail = list->tail;
939 	list->tail = index;
940 	if (old_tail == -1)
941 		list->head = index;
942 	else
943 		tx->dqo.pending_packets[old_tail].next = index;
944 
945 	pending_packet->next = -1;
946 	pending_packet->prev = old_tail;
947 }
948 
949 static void remove_from_list(struct gve_tx_ring *tx,
950 			     struct gve_index_list *list,
951 			     struct gve_tx_pending_packet_dqo *pkt)
952 {
953 	s16 prev_index, next_index;
954 
955 	prev_index = pkt->prev;
956 	next_index = pkt->next;
957 
958 	if (prev_index == -1) {
959 		/* Node is head */
960 		list->head = next_index;
961 	} else {
962 		tx->dqo.pending_packets[prev_index].next = next_index;
963 	}
964 	if (next_index == -1) {
965 		/* Node is tail */
966 		list->tail = prev_index;
967 	} else {
968 		tx->dqo.pending_packets[next_index].prev = prev_index;
969 	}
970 }
971 
972 static void gve_unmap_packet(struct device *dev,
973 			     struct gve_tx_pending_packet_dqo *pkt)
974 {
975 	int i;
976 
977 	/* SKB linear portion is guaranteed to be mapped */
978 	dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]),
979 			 dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE);
980 	for (i = 1; i < pkt->num_bufs; i++) {
981 		dma_unmap_page(dev, dma_unmap_addr(pkt, dma[i]),
982 			       dma_unmap_len(pkt, len[i]), DMA_TO_DEVICE);
983 	}
984 	pkt->num_bufs = 0;
985 }
986 
987 /* Completion types and expected behavior:
988  * No Miss compl + Packet compl = Packet completed normally.
989  * Miss compl + Re-inject compl = Packet completed normally.
990  * No Miss compl + Re-inject compl = Skipped i.e. packet not completed.
991  * Miss compl + Packet compl = Skipped i.e. packet not completed.
992  */
993 static void gve_handle_packet_completion(struct gve_priv *priv,
994 					 struct gve_tx_ring *tx, bool is_napi,
995 					 u16 compl_tag, u64 *bytes, u64 *pkts,
996 					 bool is_reinjection)
997 {
998 	struct gve_tx_pending_packet_dqo *pending_packet;
999 
1000 	if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) {
1001 		net_err_ratelimited("%s: Invalid TX completion tag: %d\n",
1002 				    priv->dev->name, (int)compl_tag);
1003 		return;
1004 	}
1005 
1006 	pending_packet = &tx->dqo.pending_packets[compl_tag];
1007 
1008 	if (unlikely(is_reinjection)) {
1009 		if (unlikely(pending_packet->state ==
1010 			     GVE_PACKET_STATE_TIMED_OUT_COMPL)) {
1011 			net_err_ratelimited("%s: Re-injection completion: %d received after timeout.\n",
1012 					    priv->dev->name, (int)compl_tag);
1013 			/* Packet was already completed as a result of timeout,
1014 			 * so just remove from list and free pending packet.
1015 			 */
1016 			remove_from_list(tx,
1017 					 &tx->dqo_compl.timed_out_completions,
1018 					 pending_packet);
1019 			gve_free_pending_packet(tx, pending_packet);
1020 			return;
1021 		}
1022 		if (unlikely(pending_packet->state !=
1023 			     GVE_PACKET_STATE_PENDING_REINJECT_COMPL)) {
1024 			/* No outstanding miss completion but packet allocated
1025 			 * implies packet receives a re-injection completion
1026 			 * without a prior miss completion. Return without
1027 			 * completing the packet.
1028 			 */
1029 			net_err_ratelimited("%s: Re-injection completion received without corresponding miss completion: %d\n",
1030 					    priv->dev->name, (int)compl_tag);
1031 			return;
1032 		}
1033 		remove_from_list(tx, &tx->dqo_compl.miss_completions,
1034 				 pending_packet);
1035 	} else {
1036 		/* Packet is allocated but not a pending data completion. */
1037 		if (unlikely(pending_packet->state !=
1038 			     GVE_PACKET_STATE_PENDING_DATA_COMPL)) {
1039 			net_err_ratelimited("%s: No pending data completion: %d\n",
1040 					    priv->dev->name, (int)compl_tag);
1041 			return;
1042 		}
1043 	}
1044 	tx->dqo_tx.completed_packet_desc_cnt += pending_packet->num_bufs;
1045 	if (tx->dqo.qpl)
1046 		gve_free_tx_qpl_bufs(tx, pending_packet);
1047 	else
1048 		gve_unmap_packet(tx->dev, pending_packet);
1049 
1050 	*bytes += pending_packet->skb->len;
1051 	(*pkts)++;
1052 	napi_consume_skb(pending_packet->skb, is_napi);
1053 	pending_packet->skb = NULL;
1054 	gve_free_pending_packet(tx, pending_packet);
1055 }
1056 
1057 static void gve_handle_miss_completion(struct gve_priv *priv,
1058 				       struct gve_tx_ring *tx, u16 compl_tag,
1059 				       u64 *bytes, u64 *pkts)
1060 {
1061 	struct gve_tx_pending_packet_dqo *pending_packet;
1062 
1063 	if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) {
1064 		net_err_ratelimited("%s: Invalid TX completion tag: %d\n",
1065 				    priv->dev->name, (int)compl_tag);
1066 		return;
1067 	}
1068 
1069 	pending_packet = &tx->dqo.pending_packets[compl_tag];
1070 	if (unlikely(pending_packet->state !=
1071 				GVE_PACKET_STATE_PENDING_DATA_COMPL)) {
1072 		net_err_ratelimited("%s: Unexpected packet state: %d for completion tag : %d\n",
1073 				    priv->dev->name, (int)pending_packet->state,
1074 				    (int)compl_tag);
1075 		return;
1076 	}
1077 
1078 	pending_packet->state = GVE_PACKET_STATE_PENDING_REINJECT_COMPL;
1079 	/* jiffies can wraparound but time comparisons can handle overflows. */
1080 	pending_packet->timeout_jiffies =
1081 			jiffies +
1082 			msecs_to_jiffies(GVE_REINJECT_COMPL_TIMEOUT *
1083 					 MSEC_PER_SEC);
1084 	add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet);
1085 
1086 	*bytes += pending_packet->skb->len;
1087 	(*pkts)++;
1088 }
1089 
1090 static void remove_miss_completions(struct gve_priv *priv,
1091 				    struct gve_tx_ring *tx)
1092 {
1093 	struct gve_tx_pending_packet_dqo *pending_packet;
1094 	s16 next_index;
1095 
1096 	next_index = tx->dqo_compl.miss_completions.head;
1097 	while (next_index != -1) {
1098 		pending_packet = &tx->dqo.pending_packets[next_index];
1099 		next_index = pending_packet->next;
1100 		/* Break early because packets should timeout in order. */
1101 		if (time_is_after_jiffies(pending_packet->timeout_jiffies))
1102 			break;
1103 
1104 		remove_from_list(tx, &tx->dqo_compl.miss_completions,
1105 				 pending_packet);
1106 		/* Unmap/free TX buffers and free skb but do not unallocate packet i.e.
1107 		 * the completion tag is not freed to ensure that the driver
1108 		 * can take appropriate action if a corresponding valid
1109 		 * completion is received later.
1110 		 */
1111 		if (tx->dqo.qpl)
1112 			gve_free_tx_qpl_bufs(tx, pending_packet);
1113 		else
1114 			gve_unmap_packet(tx->dev, pending_packet);
1115 
1116 		/* This indicates the packet was dropped. */
1117 		dev_kfree_skb_any(pending_packet->skb);
1118 		pending_packet->skb = NULL;
1119 		tx->dropped_pkt++;
1120 		net_err_ratelimited("%s: No reinjection completion was received for: %d.\n",
1121 				    priv->dev->name,
1122 				    (int)(pending_packet - tx->dqo.pending_packets));
1123 
1124 		pending_packet->state = GVE_PACKET_STATE_TIMED_OUT_COMPL;
1125 		pending_packet->timeout_jiffies =
1126 				jiffies +
1127 				msecs_to_jiffies(GVE_DEALLOCATE_COMPL_TIMEOUT *
1128 						 MSEC_PER_SEC);
1129 		/* Maintain pending packet in another list so the packet can be
1130 		 * unallocated at a later time.
1131 		 */
1132 		add_to_list(tx, &tx->dqo_compl.timed_out_completions,
1133 			    pending_packet);
1134 	}
1135 }
1136 
1137 static void remove_timed_out_completions(struct gve_priv *priv,
1138 					 struct gve_tx_ring *tx)
1139 {
1140 	struct gve_tx_pending_packet_dqo *pending_packet;
1141 	s16 next_index;
1142 
1143 	next_index = tx->dqo_compl.timed_out_completions.head;
1144 	while (next_index != -1) {
1145 		pending_packet = &tx->dqo.pending_packets[next_index];
1146 		next_index = pending_packet->next;
1147 		/* Break early because packets should timeout in order. */
1148 		if (time_is_after_jiffies(pending_packet->timeout_jiffies))
1149 			break;
1150 
1151 		remove_from_list(tx, &tx->dqo_compl.timed_out_completions,
1152 				 pending_packet);
1153 		gve_free_pending_packet(tx, pending_packet);
1154 	}
1155 }
1156 
1157 int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
1158 			  struct napi_struct *napi)
1159 {
1160 	u64 reinject_compl_bytes = 0;
1161 	u64 reinject_compl_pkts = 0;
1162 	int num_descs_cleaned = 0;
1163 	u64 miss_compl_bytes = 0;
1164 	u64 miss_compl_pkts = 0;
1165 	u64 pkt_compl_bytes = 0;
1166 	u64 pkt_compl_pkts = 0;
1167 
1168 	/* Limit in order to avoid blocking for too long */
1169 	while (!napi || pkt_compl_pkts < napi->weight) {
1170 		struct gve_tx_compl_desc *compl_desc =
1171 			&tx->dqo.compl_ring[tx->dqo_compl.head];
1172 		u16 type;
1173 
1174 		if (compl_desc->generation == tx->dqo_compl.cur_gen_bit)
1175 			break;
1176 
1177 		/* Prefetch the next descriptor. */
1178 		prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) &
1179 				tx->dqo.complq_mask]);
1180 
1181 		/* Do not read data until we own the descriptor */
1182 		dma_rmb();
1183 		type = compl_desc->type;
1184 
1185 		if (type == GVE_COMPL_TYPE_DQO_DESC) {
1186 			/* This is the last descriptor fetched by HW plus one */
1187 			u16 tx_head = le16_to_cpu(compl_desc->tx_head);
1188 
1189 			atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head);
1190 		} else if (type == GVE_COMPL_TYPE_DQO_PKT) {
1191 			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1192 			if (compl_tag & GVE_ALT_MISS_COMPL_BIT) {
1193 				compl_tag &= ~GVE_ALT_MISS_COMPL_BIT;
1194 				gve_handle_miss_completion(priv, tx, compl_tag,
1195 							   &miss_compl_bytes,
1196 							   &miss_compl_pkts);
1197 			} else {
1198 				gve_handle_packet_completion(priv, tx, !!napi,
1199 							     compl_tag,
1200 							     &pkt_compl_bytes,
1201 							     &pkt_compl_pkts,
1202 							     false);
1203 			}
1204 		} else if (type == GVE_COMPL_TYPE_DQO_MISS) {
1205 			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1206 
1207 			gve_handle_miss_completion(priv, tx, compl_tag,
1208 						   &miss_compl_bytes,
1209 						   &miss_compl_pkts);
1210 		} else if (type == GVE_COMPL_TYPE_DQO_REINJECTION) {
1211 			u16 compl_tag = le16_to_cpu(compl_desc->completion_tag);
1212 
1213 			gve_handle_packet_completion(priv, tx, !!napi,
1214 						     compl_tag,
1215 						     &reinject_compl_bytes,
1216 						     &reinject_compl_pkts,
1217 						     true);
1218 		}
1219 
1220 		tx->dqo_compl.head =
1221 			(tx->dqo_compl.head + 1) & tx->dqo.complq_mask;
1222 		/* Flip the generation bit when we wrap around */
1223 		tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0;
1224 		num_descs_cleaned++;
1225 	}
1226 
1227 	netdev_tx_completed_queue(tx->netdev_txq,
1228 				  pkt_compl_pkts + miss_compl_pkts,
1229 				  pkt_compl_bytes + miss_compl_bytes);
1230 
1231 	remove_miss_completions(priv, tx);
1232 	remove_timed_out_completions(priv, tx);
1233 
1234 	u64_stats_update_begin(&tx->statss);
1235 	tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes;
1236 	tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts;
1237 	u64_stats_update_end(&tx->statss);
1238 	return num_descs_cleaned;
1239 }
1240 
1241 bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean)
1242 {
1243 	struct gve_tx_compl_desc *compl_desc;
1244 	struct gve_tx_ring *tx = block->tx;
1245 	struct gve_priv *priv = block->priv;
1246 
1247 	if (do_clean) {
1248 		int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx,
1249 							      &block->napi);
1250 
1251 		/* Sync with queue being stopped in `gve_maybe_stop_tx_dqo()` */
1252 		mb();
1253 
1254 		if (netif_tx_queue_stopped(tx->netdev_txq) &&
1255 		    num_descs_cleaned > 0) {
1256 			tx->wake_queue++;
1257 			netif_tx_wake_queue(tx->netdev_txq);
1258 		}
1259 	}
1260 
1261 	/* Return true if we still have work. */
1262 	compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head];
1263 	return compl_desc->generation != tx->dqo_compl.cur_gen_bit;
1264 }
1265