1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2021 Google, Inc.
5  */
6 
7 #include "gve.h"
8 #include "gve_adminq.h"
9 #include "gve_utils.h"
10 #include <linux/etherdevice.h>
11 #include <linux/filter.h>
12 #include <net/xdp.h>
13 #include <net/xdp_sock_drv.h>
14 
15 static void gve_rx_free_buffer(struct device *dev,
16 			       struct gve_rx_slot_page_info *page_info,
17 			       union gve_rx_data_slot *data_slot)
18 {
19 	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
20 				      GVE_DATA_SLOT_ADDR_PAGE_MASK);
21 
22 	page_ref_sub(page_info->page, page_info->pagecnt_bias - 1);
23 	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
24 }
25 
26 static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
27 {
28 	u32 slots = rx->mask + 1;
29 	int i;
30 
31 	if (rx->data.raw_addressing) {
32 		for (i = 0; i < slots; i++)
33 			gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
34 					   &rx->data.data_ring[i]);
35 	} else {
36 		for (i = 0; i < slots; i++)
37 			page_ref_sub(rx->data.page_info[i].page,
38 				     rx->data.page_info[i].pagecnt_bias - 1);
39 		gve_unassign_qpl(priv, rx->data.qpl->id);
40 		rx->data.qpl = NULL;
41 
42 		for (i = 0; i < rx->qpl_copy_pool_mask + 1; i++) {
43 			page_ref_sub(rx->qpl_copy_pool[i].page,
44 				     rx->qpl_copy_pool[i].pagecnt_bias - 1);
45 			put_page(rx->qpl_copy_pool[i].page);
46 		}
47 	}
48 	kvfree(rx->data.page_info);
49 	rx->data.page_info = NULL;
50 }
51 
52 static void gve_rx_free_ring(struct gve_priv *priv, int idx)
53 {
54 	struct gve_rx_ring *rx = &priv->rx[idx];
55 	struct device *dev = &priv->pdev->dev;
56 	u32 slots = rx->mask + 1;
57 	size_t bytes;
58 
59 	gve_rx_remove_from_block(priv, idx);
60 
61 	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
62 	dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
63 	rx->desc.desc_ring = NULL;
64 
65 	dma_free_coherent(dev, sizeof(*rx->q_resources),
66 			  rx->q_resources, rx->q_resources_bus);
67 	rx->q_resources = NULL;
68 
69 	gve_rx_unfill_pages(priv, rx);
70 
71 	bytes = sizeof(*rx->data.data_ring) * slots;
72 	dma_free_coherent(dev, bytes, rx->data.data_ring,
73 			  rx->data.data_bus);
74 	rx->data.data_ring = NULL;
75 
76 	kvfree(rx->qpl_copy_pool);
77 	rx->qpl_copy_pool = NULL;
78 
79 	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
80 }
81 
82 static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
83 			     dma_addr_t addr, struct page *page, __be64 *slot_addr)
84 {
85 	page_info->page = page;
86 	page_info->page_offset = 0;
87 	page_info->page_address = page_address(page);
88 	*slot_addr = cpu_to_be64(addr);
89 	/* The page already has 1 ref */
90 	page_ref_add(page, INT_MAX - 1);
91 	page_info->pagecnt_bias = INT_MAX;
92 }
93 
94 static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
95 			       struct gve_rx_slot_page_info *page_info,
96 			       union gve_rx_data_slot *data_slot)
97 {
98 	struct page *page;
99 	dma_addr_t dma;
100 	int err;
101 
102 	err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE,
103 			     GFP_ATOMIC);
104 	if (err)
105 		return err;
106 
107 	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
108 	return 0;
109 }
110 
111 static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
112 {
113 	struct gve_priv *priv = rx->gve;
114 	u32 slots;
115 	int err;
116 	int i;
117 	int j;
118 
119 	/* Allocate one page per Rx queue slot. Each page is split into two
120 	 * packet buffers, when possible we "page flip" between the two.
121 	 */
122 	slots = rx->mask + 1;
123 
124 	rx->data.page_info = kvzalloc(slots *
125 				      sizeof(*rx->data.page_info), GFP_KERNEL);
126 	if (!rx->data.page_info)
127 		return -ENOMEM;
128 
129 	if (!rx->data.raw_addressing) {
130 		rx->data.qpl = gve_assign_rx_qpl(priv, rx->q_num);
131 		if (!rx->data.qpl) {
132 			kvfree(rx->data.page_info);
133 			rx->data.page_info = NULL;
134 			return -ENOMEM;
135 		}
136 	}
137 	for (i = 0; i < slots; i++) {
138 		if (!rx->data.raw_addressing) {
139 			struct page *page = rx->data.qpl->pages[i];
140 			dma_addr_t addr = i * PAGE_SIZE;
141 
142 			gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
143 					    &rx->data.data_ring[i].qpl_offset);
144 			continue;
145 		}
146 		err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
147 					  &rx->data.data_ring[i]);
148 		if (err)
149 			goto alloc_err;
150 	}
151 
152 	if (!rx->data.raw_addressing) {
153 		for (j = 0; j < rx->qpl_copy_pool_mask + 1; j++) {
154 			struct page *page = alloc_page(GFP_KERNEL);
155 
156 			if (!page) {
157 				err = -ENOMEM;
158 				goto alloc_err_qpl;
159 			}
160 
161 			rx->qpl_copy_pool[j].page = page;
162 			rx->qpl_copy_pool[j].page_offset = 0;
163 			rx->qpl_copy_pool[j].page_address = page_address(page);
164 
165 			/* The page already has 1 ref. */
166 			page_ref_add(page, INT_MAX - 1);
167 			rx->qpl_copy_pool[j].pagecnt_bias = INT_MAX;
168 		}
169 	}
170 
171 	return slots;
172 
173 alloc_err_qpl:
174 	while (j--) {
175 		page_ref_sub(rx->qpl_copy_pool[j].page,
176 			     rx->qpl_copy_pool[j].pagecnt_bias - 1);
177 		put_page(rx->qpl_copy_pool[j].page);
178 	}
179 alloc_err:
180 	while (i--)
181 		gve_rx_free_buffer(&priv->pdev->dev,
182 				   &rx->data.page_info[i],
183 				   &rx->data.data_ring[i]);
184 	return err;
185 }
186 
187 static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
188 {
189 	ctx->skb_head = NULL;
190 	ctx->skb_tail = NULL;
191 	ctx->total_size = 0;
192 	ctx->frag_cnt = 0;
193 	ctx->drop_pkt = false;
194 }
195 
196 static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
197 {
198 	struct gve_rx_ring *rx = &priv->rx[idx];
199 	struct device *hdev = &priv->pdev->dev;
200 	u32 slots, npages;
201 	int filled_pages;
202 	size_t bytes;
203 	int err;
204 
205 	netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
206 	/* Make sure everything is zeroed to start with */
207 	memset(rx, 0, sizeof(*rx));
208 
209 	rx->gve = priv;
210 	rx->q_num = idx;
211 
212 	slots = priv->rx_data_slot_cnt;
213 	rx->mask = slots - 1;
214 	rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
215 
216 	/* alloc rx data ring */
217 	bytes = sizeof(*rx->data.data_ring) * slots;
218 	rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
219 						&rx->data.data_bus,
220 						GFP_KERNEL);
221 	if (!rx->data.data_ring)
222 		return -ENOMEM;
223 
224 	rx->qpl_copy_pool_mask = min_t(u32, U32_MAX, slots * 2) - 1;
225 	rx->qpl_copy_pool_head = 0;
226 	rx->qpl_copy_pool = kvcalloc(rx->qpl_copy_pool_mask + 1,
227 				     sizeof(rx->qpl_copy_pool[0]),
228 				     GFP_KERNEL);
229 
230 	if (!rx->qpl_copy_pool) {
231 		err = -ENOMEM;
232 		goto abort_with_slots;
233 	}
234 
235 	filled_pages = gve_prefill_rx_pages(rx);
236 	if (filled_pages < 0) {
237 		err = -ENOMEM;
238 		goto abort_with_copy_pool;
239 	}
240 	rx->fill_cnt = filled_pages;
241 	/* Ensure data ring slots (packet buffers) are visible. */
242 	dma_wmb();
243 
244 	/* Alloc gve_queue_resources */
245 	rx->q_resources =
246 		dma_alloc_coherent(hdev,
247 				   sizeof(*rx->q_resources),
248 				   &rx->q_resources_bus,
249 				   GFP_KERNEL);
250 	if (!rx->q_resources) {
251 		err = -ENOMEM;
252 		goto abort_filled;
253 	}
254 	netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
255 		  (unsigned long)rx->data.data_bus);
256 
257 	/* alloc rx desc ring */
258 	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
259 	npages = bytes / PAGE_SIZE;
260 	if (npages * PAGE_SIZE != bytes) {
261 		err = -EIO;
262 		goto abort_with_q_resources;
263 	}
264 
265 	rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
266 						GFP_KERNEL);
267 	if (!rx->desc.desc_ring) {
268 		err = -ENOMEM;
269 		goto abort_with_q_resources;
270 	}
271 	rx->cnt = 0;
272 	rx->db_threshold = priv->rx_desc_cnt / 2;
273 	rx->desc.seqno = 1;
274 
275 	/* Allocating half-page buffers allows page-flipping which is faster
276 	 * than copying or allocating new pages.
277 	 */
278 	rx->packet_buffer_size = PAGE_SIZE / 2;
279 	gve_rx_ctx_clear(&rx->ctx);
280 	gve_rx_add_to_block(priv, idx);
281 
282 	return 0;
283 
284 abort_with_q_resources:
285 	dma_free_coherent(hdev, sizeof(*rx->q_resources),
286 			  rx->q_resources, rx->q_resources_bus);
287 	rx->q_resources = NULL;
288 abort_filled:
289 	gve_rx_unfill_pages(priv, rx);
290 abort_with_copy_pool:
291 	kvfree(rx->qpl_copy_pool);
292 	rx->qpl_copy_pool = NULL;
293 abort_with_slots:
294 	bytes = sizeof(*rx->data.data_ring) * slots;
295 	dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
296 	rx->data.data_ring = NULL;
297 
298 	return err;
299 }
300 
301 int gve_rx_alloc_rings(struct gve_priv *priv)
302 {
303 	int err = 0;
304 	int i;
305 
306 	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
307 		err = gve_rx_alloc_ring(priv, i);
308 		if (err) {
309 			netif_err(priv, drv, priv->dev,
310 				  "Failed to alloc rx ring=%d: err=%d\n",
311 				  i, err);
312 			break;
313 		}
314 	}
315 	/* Unallocate if there was an error */
316 	if (err) {
317 		int j;
318 
319 		for (j = 0; j < i; j++)
320 			gve_rx_free_ring(priv, j);
321 	}
322 	return err;
323 }
324 
325 void gve_rx_free_rings_gqi(struct gve_priv *priv)
326 {
327 	int i;
328 
329 	for (i = 0; i < priv->rx_cfg.num_queues; i++)
330 		gve_rx_free_ring(priv, i);
331 }
332 
333 void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
334 {
335 	u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
336 
337 	iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
338 }
339 
340 static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
341 {
342 	if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
343 		return PKT_HASH_TYPE_L4;
344 	if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
345 		return PKT_HASH_TYPE_L3;
346 	return PKT_HASH_TYPE_L2;
347 }
348 
349 static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
350 					struct gve_rx_slot_page_info *page_info,
351 					u16 packet_buffer_size, u16 len,
352 					struct gve_rx_ctx *ctx)
353 {
354 	u32 offset = page_info->page_offset + page_info->pad;
355 	struct sk_buff *skb = ctx->skb_tail;
356 	int num_frags = 0;
357 
358 	if (!skb) {
359 		skb = napi_get_frags(napi);
360 		if (unlikely(!skb))
361 			return NULL;
362 
363 		ctx->skb_head = skb;
364 		ctx->skb_tail = skb;
365 	} else {
366 		num_frags = skb_shinfo(ctx->skb_tail)->nr_frags;
367 		if (num_frags == MAX_SKB_FRAGS) {
368 			skb = napi_alloc_skb(napi, 0);
369 			if (!skb)
370 				return NULL;
371 
372 			// We will never chain more than two SKBs: 2 * 16 * 2k > 64k
373 			// which is why we do not need to chain by using skb->next
374 			skb_shinfo(ctx->skb_tail)->frag_list = skb;
375 
376 			ctx->skb_tail = skb;
377 			num_frags = 0;
378 		}
379 	}
380 
381 	if (skb != ctx->skb_head) {
382 		ctx->skb_head->len += len;
383 		ctx->skb_head->data_len += len;
384 		ctx->skb_head->truesize += packet_buffer_size;
385 	}
386 	skb_add_rx_frag(skb, num_frags, page_info->page,
387 			offset, len, packet_buffer_size);
388 
389 	return ctx->skb_head;
390 }
391 
392 static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
393 {
394 	const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);
395 
396 	/* "flip" to other packet buffer on this page */
397 	page_info->page_offset ^= PAGE_SIZE / 2;
398 	*(slot_addr) ^= offset;
399 }
400 
401 static int gve_rx_can_recycle_buffer(struct gve_rx_slot_page_info *page_info)
402 {
403 	int pagecount = page_count(page_info->page);
404 
405 	/* This page is not being used by any SKBs - reuse */
406 	if (pagecount == page_info->pagecnt_bias)
407 		return 1;
408 	/* This page is still being used by an SKB - we can't reuse */
409 	else if (pagecount > page_info->pagecnt_bias)
410 		return 0;
411 	WARN(pagecount < page_info->pagecnt_bias,
412 	     "Pagecount should never be less than the bias.");
413 	return -1;
414 }
415 
416 static struct sk_buff *
417 gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
418 		      struct gve_rx_slot_page_info *page_info, u16 len,
419 		      struct napi_struct *napi,
420 		      union gve_rx_data_slot *data_slot,
421 		      u16 packet_buffer_size, struct gve_rx_ctx *ctx)
422 {
423 	struct sk_buff *skb = gve_rx_add_frags(napi, page_info, packet_buffer_size, len, ctx);
424 
425 	if (!skb)
426 		return NULL;
427 
428 	/* Optimistically stop the kernel from freeing the page.
429 	 * We will check again in refill to determine if we need to alloc a
430 	 * new page.
431 	 */
432 	gve_dec_pagecnt_bias(page_info);
433 
434 	return skb;
435 }
436 
437 static struct sk_buff *gve_rx_copy_to_pool(struct gve_rx_ring *rx,
438 					   struct gve_rx_slot_page_info *page_info,
439 					   u16 len, struct napi_struct *napi)
440 {
441 	u32 pool_idx = rx->qpl_copy_pool_head & rx->qpl_copy_pool_mask;
442 	void *src = page_info->page_address + page_info->page_offset;
443 	struct gve_rx_slot_page_info *copy_page_info;
444 	struct gve_rx_ctx *ctx = &rx->ctx;
445 	bool alloc_page = false;
446 	struct sk_buff *skb;
447 	void *dst;
448 
449 	copy_page_info = &rx->qpl_copy_pool[pool_idx];
450 	if (!copy_page_info->can_flip) {
451 		int recycle = gve_rx_can_recycle_buffer(copy_page_info);
452 
453 		if (unlikely(recycle < 0)) {
454 			gve_schedule_reset(rx->gve);
455 			return NULL;
456 		}
457 		alloc_page = !recycle;
458 	}
459 
460 	if (alloc_page) {
461 		struct gve_rx_slot_page_info alloc_page_info;
462 		struct page *page;
463 
464 		/* The least recently used page turned out to be
465 		 * still in use by the kernel. Ignoring it and moving
466 		 * on alleviates head-of-line blocking.
467 		 */
468 		rx->qpl_copy_pool_head++;
469 
470 		page = alloc_page(GFP_ATOMIC);
471 		if (!page)
472 			return NULL;
473 
474 		alloc_page_info.page = page;
475 		alloc_page_info.page_offset = 0;
476 		alloc_page_info.page_address = page_address(page);
477 		alloc_page_info.pad = page_info->pad;
478 
479 		memcpy(alloc_page_info.page_address, src, page_info->pad + len);
480 		skb = gve_rx_add_frags(napi, &alloc_page_info,
481 				       rx->packet_buffer_size,
482 				       len, ctx);
483 
484 		u64_stats_update_begin(&rx->statss);
485 		rx->rx_frag_copy_cnt++;
486 		rx->rx_frag_alloc_cnt++;
487 		u64_stats_update_end(&rx->statss);
488 
489 		return skb;
490 	}
491 
492 	dst = copy_page_info->page_address + copy_page_info->page_offset;
493 	memcpy(dst, src, page_info->pad + len);
494 	copy_page_info->pad = page_info->pad;
495 
496 	skb = gve_rx_add_frags(napi, copy_page_info,
497 			       rx->packet_buffer_size, len, ctx);
498 	if (unlikely(!skb))
499 		return NULL;
500 
501 	gve_dec_pagecnt_bias(copy_page_info);
502 	copy_page_info->page_offset += rx->packet_buffer_size;
503 	copy_page_info->page_offset &= (PAGE_SIZE - 1);
504 
505 	if (copy_page_info->can_flip) {
506 		/* We have used both halves of this copy page, it
507 		 * is time for it to go to the back of the queue.
508 		 */
509 		copy_page_info->can_flip = false;
510 		rx->qpl_copy_pool_head++;
511 		prefetch(rx->qpl_copy_pool[rx->qpl_copy_pool_head & rx->qpl_copy_pool_mask].page);
512 	} else {
513 		copy_page_info->can_flip = true;
514 	}
515 
516 	u64_stats_update_begin(&rx->statss);
517 	rx->rx_frag_copy_cnt++;
518 	u64_stats_update_end(&rx->statss);
519 
520 	return skb;
521 }
522 
523 static struct sk_buff *
524 gve_rx_qpl(struct device *dev, struct net_device *netdev,
525 	   struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
526 	   u16 len, struct napi_struct *napi,
527 	   union gve_rx_data_slot *data_slot)
528 {
529 	struct gve_rx_ctx *ctx = &rx->ctx;
530 	struct sk_buff *skb;
531 
532 	/* if raw_addressing mode is not enabled gvnic can only receive into
533 	 * registered segments. If the buffer can't be recycled, our only
534 	 * choice is to copy the data out of it so that we can return it to the
535 	 * device.
536 	 */
537 	if (page_info->can_flip) {
538 		skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
539 		/* No point in recycling if we didn't get the skb */
540 		if (skb) {
541 			/* Make sure that the page isn't freed. */
542 			gve_dec_pagecnt_bias(page_info);
543 			gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
544 		}
545 	} else {
546 		skb = gve_rx_copy_to_pool(rx, page_info, len, napi);
547 	}
548 	return skb;
549 }
550 
551 static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
552 				  struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
553 				  u16 len, union gve_rx_data_slot *data_slot,
554 				  bool is_only_frag)
555 {
556 	struct net_device *netdev = priv->dev;
557 	struct gve_rx_ctx *ctx = &rx->ctx;
558 	struct sk_buff *skb = NULL;
559 
560 	if (len <= priv->rx_copybreak && is_only_frag)  {
561 		/* Just copy small packets */
562 		skb = gve_rx_copy(netdev, napi, page_info, len);
563 		if (skb) {
564 			u64_stats_update_begin(&rx->statss);
565 			rx->rx_copied_pkt++;
566 			rx->rx_frag_copy_cnt++;
567 			rx->rx_copybreak_pkt++;
568 			u64_stats_update_end(&rx->statss);
569 		}
570 	} else {
571 		int recycle = gve_rx_can_recycle_buffer(page_info);
572 
573 		if (unlikely(recycle < 0)) {
574 			gve_schedule_reset(priv);
575 			return NULL;
576 		}
577 		page_info->can_flip = recycle;
578 		if (page_info->can_flip) {
579 			u64_stats_update_begin(&rx->statss);
580 			rx->rx_frag_flip_cnt++;
581 			u64_stats_update_end(&rx->statss);
582 		}
583 
584 		if (rx->data.raw_addressing) {
585 			skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
586 						    page_info, len, napi,
587 						    data_slot,
588 						    rx->packet_buffer_size, ctx);
589 		} else {
590 			skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
591 					 page_info, len, napi, data_slot);
592 		}
593 	}
594 	return skb;
595 }
596 
597 static int gve_xsk_pool_redirect(struct net_device *dev,
598 				 struct gve_rx_ring *rx,
599 				 void *data, int len,
600 				 struct bpf_prog *xdp_prog)
601 {
602 	struct xdp_buff *xdp;
603 	int err;
604 
605 	if (rx->xsk_pool->frame_len < len)
606 		return -E2BIG;
607 	xdp = xsk_buff_alloc(rx->xsk_pool);
608 	if (!xdp) {
609 		u64_stats_update_begin(&rx->statss);
610 		rx->xdp_alloc_fails++;
611 		u64_stats_update_end(&rx->statss);
612 		return -ENOMEM;
613 	}
614 	xdp->data_end = xdp->data + len;
615 	memcpy(xdp->data, data, len);
616 	err = xdp_do_redirect(dev, xdp, xdp_prog);
617 	if (err)
618 		xsk_buff_free(xdp);
619 	return err;
620 }
621 
622 static int gve_xdp_redirect(struct net_device *dev, struct gve_rx_ring *rx,
623 			    struct xdp_buff *orig, struct bpf_prog *xdp_prog)
624 {
625 	int total_len, len = orig->data_end - orig->data;
626 	int headroom = XDP_PACKET_HEADROOM;
627 	struct xdp_buff new;
628 	void *frame;
629 	int err;
630 
631 	if (rx->xsk_pool)
632 		return gve_xsk_pool_redirect(dev, rx, orig->data,
633 					     len, xdp_prog);
634 
635 	total_len = headroom + SKB_DATA_ALIGN(len) +
636 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
637 	frame = page_frag_alloc(&rx->page_cache, total_len, GFP_ATOMIC);
638 	if (!frame) {
639 		u64_stats_update_begin(&rx->statss);
640 		rx->xdp_alloc_fails++;
641 		u64_stats_update_end(&rx->statss);
642 		return -ENOMEM;
643 	}
644 	xdp_init_buff(&new, total_len, &rx->xdp_rxq);
645 	xdp_prepare_buff(&new, frame, headroom, len, false);
646 	memcpy(new.data, orig->data, len);
647 
648 	err = xdp_do_redirect(dev, &new, xdp_prog);
649 	if (err)
650 		page_frag_free(frame);
651 
652 	return err;
653 }
654 
655 static void gve_xdp_done(struct gve_priv *priv, struct gve_rx_ring *rx,
656 			 struct xdp_buff *xdp, struct bpf_prog *xprog,
657 			 int xdp_act)
658 {
659 	struct gve_tx_ring *tx;
660 	int tx_qid;
661 	int err;
662 
663 	switch (xdp_act) {
664 	case XDP_ABORTED:
665 	case XDP_DROP:
666 	default:
667 		break;
668 	case XDP_TX:
669 		tx_qid = gve_xdp_tx_queue_id(priv, rx->q_num);
670 		tx = &priv->tx[tx_qid];
671 		spin_lock(&tx->xdp_lock);
672 		err = gve_xdp_xmit_one(priv, tx, xdp->data,
673 				       xdp->data_end - xdp->data, NULL);
674 		spin_unlock(&tx->xdp_lock);
675 
676 		if (unlikely(err)) {
677 			u64_stats_update_begin(&rx->statss);
678 			rx->xdp_tx_errors++;
679 			u64_stats_update_end(&rx->statss);
680 		}
681 		break;
682 	case XDP_REDIRECT:
683 		err = gve_xdp_redirect(priv->dev, rx, xdp, xprog);
684 
685 		if (unlikely(err)) {
686 			u64_stats_update_begin(&rx->statss);
687 			rx->xdp_redirect_errors++;
688 			u64_stats_update_end(&rx->statss);
689 		}
690 		break;
691 	}
692 	u64_stats_update_begin(&rx->statss);
693 	if ((u32)xdp_act < GVE_XDP_ACTIONS)
694 		rx->xdp_actions[xdp_act]++;
695 	u64_stats_update_end(&rx->statss);
696 }
697 
698 #define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
699 static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
700 		   struct gve_rx_desc *desc, u32 idx,
701 		   struct gve_rx_cnts *cnts)
702 {
703 	bool is_last_frag = !GVE_PKTCONT_BIT_IS_SET(desc->flags_seq);
704 	struct gve_rx_slot_page_info *page_info;
705 	u16 frag_size = be16_to_cpu(desc->len);
706 	struct gve_rx_ctx *ctx = &rx->ctx;
707 	union gve_rx_data_slot *data_slot;
708 	struct gve_priv *priv = rx->gve;
709 	struct sk_buff *skb = NULL;
710 	struct bpf_prog *xprog;
711 	struct xdp_buff xdp;
712 	dma_addr_t page_bus;
713 	void *va;
714 
715 	u16 len = frag_size;
716 	struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
717 	bool is_first_frag = ctx->frag_cnt == 0;
718 
719 	bool is_only_frag = is_first_frag && is_last_frag;
720 
721 	if (unlikely(ctx->drop_pkt))
722 		goto finish_frag;
723 
724 	if (desc->flags_seq & GVE_RXF_ERR) {
725 		ctx->drop_pkt = true;
726 		cnts->desc_err_pkt_cnt++;
727 		napi_free_frags(napi);
728 		goto finish_frag;
729 	}
730 
731 	if (unlikely(frag_size > rx->packet_buffer_size)) {
732 		netdev_warn(priv->dev, "Unexpected frag size %d, can't exceed %d, scheduling reset",
733 			    frag_size, rx->packet_buffer_size);
734 		ctx->drop_pkt = true;
735 		napi_free_frags(napi);
736 		gve_schedule_reset(rx->gve);
737 		goto finish_frag;
738 	}
739 
740 	/* Prefetch two packet buffers ahead, we will need it soon. */
741 	page_info = &rx->data.page_info[(idx + 2) & rx->mask];
742 	va = page_info->page_address + page_info->page_offset;
743 	prefetch(page_info->page); /* Kernel page struct. */
744 	prefetch(va);              /* Packet header. */
745 	prefetch(va + 64);         /* Next cacheline too. */
746 
747 	page_info = &rx->data.page_info[idx];
748 	data_slot = &rx->data.data_ring[idx];
749 	page_bus = (rx->data.raw_addressing) ?
750 		be64_to_cpu(data_slot->addr) - page_info->page_offset :
751 		rx->data.qpl->page_buses[idx];
752 	dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
753 				PAGE_SIZE, DMA_FROM_DEVICE);
754 	page_info->pad = is_first_frag ? GVE_RX_PAD : 0;
755 	len -= page_info->pad;
756 	frag_size -= page_info->pad;
757 
758 	xprog = READ_ONCE(priv->xdp_prog);
759 	if (xprog && is_only_frag) {
760 		void *old_data;
761 		int xdp_act;
762 
763 		xdp_init_buff(&xdp, rx->packet_buffer_size, &rx->xdp_rxq);
764 		xdp_prepare_buff(&xdp, page_info->page_address +
765 				 page_info->page_offset, GVE_RX_PAD,
766 				 len, false);
767 		old_data = xdp.data;
768 		xdp_act = bpf_prog_run_xdp(xprog, &xdp);
769 		if (xdp_act != XDP_PASS) {
770 			gve_xdp_done(priv, rx, &xdp, xprog, xdp_act);
771 			ctx->total_size += frag_size;
772 			goto finish_ok_pkt;
773 		}
774 
775 		page_info->pad += xdp.data - old_data;
776 		len = xdp.data_end - xdp.data;
777 
778 		u64_stats_update_begin(&rx->statss);
779 		rx->xdp_actions[XDP_PASS]++;
780 		u64_stats_update_end(&rx->statss);
781 	}
782 
783 	skb = gve_rx_skb(priv, rx, page_info, napi, len,
784 			 data_slot, is_only_frag);
785 	if (!skb) {
786 		u64_stats_update_begin(&rx->statss);
787 		rx->rx_skb_alloc_fail++;
788 		u64_stats_update_end(&rx->statss);
789 
790 		napi_free_frags(napi);
791 		ctx->drop_pkt = true;
792 		goto finish_frag;
793 	}
794 	ctx->total_size += frag_size;
795 
796 	if (is_first_frag) {
797 		if (likely(feat & NETIF_F_RXCSUM)) {
798 			/* NIC passes up the partial sum */
799 			if (desc->csum)
800 				skb->ip_summed = CHECKSUM_COMPLETE;
801 			else
802 				skb->ip_summed = CHECKSUM_NONE;
803 			skb->csum = csum_unfold(desc->csum);
804 		}
805 
806 		/* parse flags & pass relevant info up */
807 		if (likely(feat & NETIF_F_RXHASH) &&
808 		    gve_needs_rss(desc->flags_seq))
809 			skb_set_hash(skb, be32_to_cpu(desc->rss_hash),
810 				     gve_rss_type(desc->flags_seq));
811 	}
812 
813 	if (is_last_frag) {
814 		skb_record_rx_queue(skb, rx->q_num);
815 		if (skb_is_nonlinear(skb))
816 			napi_gro_frags(napi);
817 		else
818 			napi_gro_receive(napi, skb);
819 		goto finish_ok_pkt;
820 	}
821 
822 	goto finish_frag;
823 
824 finish_ok_pkt:
825 	cnts->ok_pkt_bytes += ctx->total_size;
826 	cnts->ok_pkt_cnt++;
827 finish_frag:
828 	ctx->frag_cnt++;
829 	if (is_last_frag) {
830 		cnts->total_pkt_cnt++;
831 		cnts->cont_pkt_cnt += (ctx->frag_cnt > 1);
832 		gve_rx_ctx_clear(ctx);
833 	}
834 }
835 
836 bool gve_rx_work_pending(struct gve_rx_ring *rx)
837 {
838 	struct gve_rx_desc *desc;
839 	__be16 flags_seq;
840 	u32 next_idx;
841 
842 	next_idx = rx->cnt & rx->mask;
843 	desc = rx->desc.desc_ring + next_idx;
844 
845 	flags_seq = desc->flags_seq;
846 
847 	return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
848 }
849 
850 static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
851 {
852 	int refill_target = rx->mask + 1;
853 	u32 fill_cnt = rx->fill_cnt;
854 
855 	while (fill_cnt - rx->cnt < refill_target) {
856 		struct gve_rx_slot_page_info *page_info;
857 		u32 idx = fill_cnt & rx->mask;
858 
859 		page_info = &rx->data.page_info[idx];
860 		if (page_info->can_flip) {
861 			/* The other half of the page is free because it was
862 			 * free when we processed the descriptor. Flip to it.
863 			 */
864 			union gve_rx_data_slot *data_slot =
865 						&rx->data.data_ring[idx];
866 
867 			gve_rx_flip_buff(page_info, &data_slot->addr);
868 			page_info->can_flip = 0;
869 		} else {
870 			/* It is possible that the networking stack has already
871 			 * finished processing all outstanding packets in the buffer
872 			 * and it can be reused.
873 			 * Flipping is unnecessary here - if the networking stack still
874 			 * owns half the page it is impossible to tell which half. Either
875 			 * the whole page is free or it needs to be replaced.
876 			 */
877 			int recycle = gve_rx_can_recycle_buffer(page_info);
878 
879 			if (recycle < 0) {
880 				if (!rx->data.raw_addressing)
881 					gve_schedule_reset(priv);
882 				return false;
883 			}
884 			if (!recycle) {
885 				/* We can't reuse the buffer - alloc a new one*/
886 				union gve_rx_data_slot *data_slot =
887 						&rx->data.data_ring[idx];
888 				struct device *dev = &priv->pdev->dev;
889 				gve_rx_free_buffer(dev, page_info, data_slot);
890 				page_info->page = NULL;
891 				if (gve_rx_alloc_buffer(priv, dev, page_info,
892 							data_slot)) {
893 					u64_stats_update_begin(&rx->statss);
894 					rx->rx_buf_alloc_fail++;
895 					u64_stats_update_end(&rx->statss);
896 					break;
897 				}
898 			}
899 		}
900 		fill_cnt++;
901 	}
902 	rx->fill_cnt = fill_cnt;
903 	return true;
904 }
905 
906 static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
907 			     netdev_features_t feat)
908 {
909 	u64 xdp_redirects = rx->xdp_actions[XDP_REDIRECT];
910 	u64 xdp_txs = rx->xdp_actions[XDP_TX];
911 	struct gve_rx_ctx *ctx = &rx->ctx;
912 	struct gve_priv *priv = rx->gve;
913 	struct gve_rx_cnts cnts = {0};
914 	struct gve_rx_desc *next_desc;
915 	u32 idx = rx->cnt & rx->mask;
916 	u32 work_done = 0;
917 
918 	struct gve_rx_desc *desc = &rx->desc.desc_ring[idx];
919 
920 	// Exceed budget only if (and till) the inflight packet is consumed.
921 	while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
922 	       (work_done < budget || ctx->frag_cnt)) {
923 		next_desc = &rx->desc.desc_ring[(idx + 1) & rx->mask];
924 		prefetch(next_desc);
925 
926 		gve_rx(rx, feat, desc, idx, &cnts);
927 
928 		rx->cnt++;
929 		idx = rx->cnt & rx->mask;
930 		desc = &rx->desc.desc_ring[idx];
931 		rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
932 		work_done++;
933 	}
934 
935 	// The device will only send whole packets.
936 	if (unlikely(ctx->frag_cnt)) {
937 		struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
938 
939 		napi_free_frags(napi);
940 		gve_rx_ctx_clear(&rx->ctx);
941 		netdev_warn(priv->dev, "Unexpected seq number %d with incomplete packet, expected %d, scheduling reset",
942 			    GVE_SEQNO(desc->flags_seq), rx->desc.seqno);
943 		gve_schedule_reset(rx->gve);
944 	}
945 
946 	if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
947 		return 0;
948 
949 	if (work_done) {
950 		u64_stats_update_begin(&rx->statss);
951 		rx->rpackets += cnts.ok_pkt_cnt;
952 		rx->rbytes += cnts.ok_pkt_bytes;
953 		rx->rx_cont_packet_cnt += cnts.cont_pkt_cnt;
954 		rx->rx_desc_err_dropped_pkt += cnts.desc_err_pkt_cnt;
955 		u64_stats_update_end(&rx->statss);
956 	}
957 
958 	if (xdp_txs != rx->xdp_actions[XDP_TX])
959 		gve_xdp_tx_flush(priv, rx->q_num);
960 
961 	if (xdp_redirects != rx->xdp_actions[XDP_REDIRECT])
962 		xdp_do_flush();
963 
964 	/* restock ring slots */
965 	if (!rx->data.raw_addressing) {
966 		/* In QPL mode buffs are refilled as the desc are processed */
967 		rx->fill_cnt += work_done;
968 	} else if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
969 		/* In raw addressing mode buffs are only refilled if the avail
970 		 * falls below a threshold.
971 		 */
972 		if (!gve_rx_refill_buffers(priv, rx))
973 			return 0;
974 
975 		/* If we were not able to completely refill buffers, we'll want
976 		 * to schedule this queue for work again to refill buffers.
977 		 */
978 		if (rx->fill_cnt - rx->cnt <= rx->db_threshold) {
979 			gve_rx_write_doorbell(priv, rx);
980 			return budget;
981 		}
982 	}
983 
984 	gve_rx_write_doorbell(priv, rx);
985 	return cnts.total_pkt_cnt;
986 }
987 
988 int gve_rx_poll(struct gve_notify_block *block, int budget)
989 {
990 	struct gve_rx_ring *rx = block->rx;
991 	netdev_features_t feat;
992 	int work_done = 0;
993 
994 	feat = block->napi.dev->features;
995 
996 	/* If budget is 0, do all the work */
997 	if (budget == 0)
998 		budget = INT_MAX;
999 
1000 	if (budget > 0)
1001 		work_done = gve_clean_rx_done(rx, budget, feat);
1002 
1003 	return work_done;
1004 }
1005