1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include "gve.h" 8 #include "gve_dqo.h" 9 #include "gve_adminq.h" 10 #include "gve_utils.h" 11 #include <linux/ip.h> 12 #include <linux/ipv6.h> 13 #include <linux/skbuff.h> 14 #include <linux/slab.h> 15 #include <net/ip6_checksum.h> 16 #include <net/ipv6.h> 17 #include <net/tcp.h> 18 19 static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs) 20 { 21 return page_count(bs->page_info.page) - bs->page_info.pagecnt_bias; 22 } 23 24 static void gve_free_page_dqo(struct gve_priv *priv, 25 struct gve_rx_buf_state_dqo *bs) 26 { 27 page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1); 28 gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr, 29 DMA_FROM_DEVICE); 30 bs->page_info.page = NULL; 31 } 32 33 static struct gve_rx_buf_state_dqo *gve_alloc_buf_state(struct gve_rx_ring *rx) 34 { 35 struct gve_rx_buf_state_dqo *buf_state; 36 s16 buffer_id; 37 38 buffer_id = rx->dqo.free_buf_states; 39 if (unlikely(buffer_id == -1)) 40 return NULL; 41 42 buf_state = &rx->dqo.buf_states[buffer_id]; 43 44 /* Remove buf_state from free list */ 45 rx->dqo.free_buf_states = buf_state->next; 46 47 /* Point buf_state to itself to mark it as allocated */ 48 buf_state->next = buffer_id; 49 50 return buf_state; 51 } 52 53 static bool gve_buf_state_is_allocated(struct gve_rx_ring *rx, 54 struct gve_rx_buf_state_dqo *buf_state) 55 { 56 s16 buffer_id = buf_state - rx->dqo.buf_states; 57 58 return buf_state->next == buffer_id; 59 } 60 61 static void gve_free_buf_state(struct gve_rx_ring *rx, 62 struct gve_rx_buf_state_dqo *buf_state) 63 { 64 s16 buffer_id = buf_state - rx->dqo.buf_states; 65 66 buf_state->next = rx->dqo.free_buf_states; 67 rx->dqo.free_buf_states = buffer_id; 68 } 69 70 static struct gve_rx_buf_state_dqo * 71 gve_dequeue_buf_state(struct gve_rx_ring *rx, struct gve_index_list *list) 72 { 73 struct gve_rx_buf_state_dqo *buf_state; 74 s16 buffer_id; 75 76 buffer_id = list->head; 77 if (unlikely(buffer_id == -1)) 78 return NULL; 79 80 buf_state = &rx->dqo.buf_states[buffer_id]; 81 82 /* Remove buf_state from list */ 83 list->head = buf_state->next; 84 if (buf_state->next == -1) 85 list->tail = -1; 86 87 /* Point buf_state to itself to mark it as allocated */ 88 buf_state->next = buffer_id; 89 90 return buf_state; 91 } 92 93 static void gve_enqueue_buf_state(struct gve_rx_ring *rx, 94 struct gve_index_list *list, 95 struct gve_rx_buf_state_dqo *buf_state) 96 { 97 s16 buffer_id = buf_state - rx->dqo.buf_states; 98 99 buf_state->next = -1; 100 101 if (list->head == -1) { 102 list->head = buffer_id; 103 list->tail = buffer_id; 104 } else { 105 int tail = list->tail; 106 107 rx->dqo.buf_states[tail].next = buffer_id; 108 list->tail = buffer_id; 109 } 110 } 111 112 static struct gve_rx_buf_state_dqo * 113 gve_get_recycled_buf_state(struct gve_rx_ring *rx) 114 { 115 struct gve_rx_buf_state_dqo *buf_state; 116 int i; 117 118 /* Recycled buf states are immediately usable. */ 119 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.recycled_buf_states); 120 if (likely(buf_state)) 121 return buf_state; 122 123 if (unlikely(rx->dqo.used_buf_states.head == -1)) 124 return NULL; 125 126 /* Used buf states are only usable when ref count reaches 0, which means 127 * no SKBs refer to them. 128 * 129 * Search a limited number before giving up. 130 */ 131 for (i = 0; i < 5; i++) { 132 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); 133 if (gve_buf_ref_cnt(buf_state) == 0) 134 return buf_state; 135 136 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 137 } 138 139 /* If there are no free buf states discard an entry from 140 * `used_buf_states` so it can be used. 141 */ 142 if (unlikely(rx->dqo.free_buf_states == -1)) { 143 buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states); 144 if (gve_buf_ref_cnt(buf_state) == 0) 145 return buf_state; 146 147 gve_free_page_dqo(rx->gve, buf_state); 148 gve_free_buf_state(rx, buf_state); 149 } 150 151 return NULL; 152 } 153 154 static int gve_alloc_page_dqo(struct gve_priv *priv, 155 struct gve_rx_buf_state_dqo *buf_state) 156 { 157 int err; 158 159 err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page, 160 &buf_state->addr, DMA_FROM_DEVICE); 161 if (err) 162 return err; 163 164 buf_state->page_info.page_offset = 0; 165 buf_state->page_info.page_address = 166 page_address(buf_state->page_info.page); 167 buf_state->last_single_ref_offset = 0; 168 169 /* The page already has 1 ref. */ 170 page_ref_add(buf_state->page_info.page, INT_MAX - 1); 171 buf_state->page_info.pagecnt_bias = INT_MAX; 172 173 return 0; 174 } 175 176 static void gve_rx_free_ring_dqo(struct gve_priv *priv, int idx) 177 { 178 struct gve_rx_ring *rx = &priv->rx[idx]; 179 struct device *hdev = &priv->pdev->dev; 180 size_t completion_queue_slots; 181 size_t buffer_queue_slots; 182 size_t size; 183 int i; 184 185 completion_queue_slots = rx->dqo.complq.mask + 1; 186 buffer_queue_slots = rx->dqo.bufq.mask + 1; 187 188 gve_rx_remove_from_block(priv, idx); 189 190 if (rx->q_resources) { 191 dma_free_coherent(hdev, sizeof(*rx->q_resources), 192 rx->q_resources, rx->q_resources_bus); 193 rx->q_resources = NULL; 194 } 195 196 for (i = 0; i < rx->dqo.num_buf_states; i++) { 197 struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i]; 198 199 if (bs->page_info.page) 200 gve_free_page_dqo(priv, bs); 201 } 202 203 if (rx->dqo.bufq.desc_ring) { 204 size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots; 205 dma_free_coherent(hdev, size, rx->dqo.bufq.desc_ring, 206 rx->dqo.bufq.bus); 207 rx->dqo.bufq.desc_ring = NULL; 208 } 209 210 if (rx->dqo.complq.desc_ring) { 211 size = sizeof(rx->dqo.complq.desc_ring[0]) * 212 completion_queue_slots; 213 dma_free_coherent(hdev, size, rx->dqo.complq.desc_ring, 214 rx->dqo.complq.bus); 215 rx->dqo.complq.desc_ring = NULL; 216 } 217 218 kvfree(rx->dqo.buf_states); 219 rx->dqo.buf_states = NULL; 220 221 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx); 222 } 223 224 static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx) 225 { 226 struct gve_rx_ring *rx = &priv->rx[idx]; 227 struct device *hdev = &priv->pdev->dev; 228 size_t size; 229 int i; 230 231 const u32 buffer_queue_slots = 232 priv->options_dqo_rda.rx_buff_ring_entries; 233 const u32 completion_queue_slots = priv->rx_desc_cnt; 234 235 netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n"); 236 237 memset(rx, 0, sizeof(*rx)); 238 rx->gve = priv; 239 rx->q_num = idx; 240 rx->dqo.bufq.mask = buffer_queue_slots - 1; 241 rx->dqo.complq.num_free_slots = completion_queue_slots; 242 rx->dqo.complq.mask = completion_queue_slots - 1; 243 rx->skb_head = NULL; 244 rx->skb_tail = NULL; 245 246 rx->dqo.num_buf_states = min_t(s16, S16_MAX, buffer_queue_slots * 4); 247 rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states, 248 sizeof(rx->dqo.buf_states[0]), 249 GFP_KERNEL); 250 if (!rx->dqo.buf_states) 251 return -ENOMEM; 252 253 /* Set up linked list of buffer IDs */ 254 for (i = 0; i < rx->dqo.num_buf_states - 1; i++) 255 rx->dqo.buf_states[i].next = i + 1; 256 257 rx->dqo.buf_states[rx->dqo.num_buf_states - 1].next = -1; 258 rx->dqo.recycled_buf_states.head = -1; 259 rx->dqo.recycled_buf_states.tail = -1; 260 rx->dqo.used_buf_states.head = -1; 261 rx->dqo.used_buf_states.tail = -1; 262 263 /* Allocate RX completion queue */ 264 size = sizeof(rx->dqo.complq.desc_ring[0]) * 265 completion_queue_slots; 266 rx->dqo.complq.desc_ring = 267 dma_alloc_coherent(hdev, size, &rx->dqo.complq.bus, GFP_KERNEL); 268 if (!rx->dqo.complq.desc_ring) 269 goto err; 270 271 /* Allocate RX buffer queue */ 272 size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots; 273 rx->dqo.bufq.desc_ring = 274 dma_alloc_coherent(hdev, size, &rx->dqo.bufq.bus, GFP_KERNEL); 275 if (!rx->dqo.bufq.desc_ring) 276 goto err; 277 278 rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources), 279 &rx->q_resources_bus, GFP_KERNEL); 280 if (!rx->q_resources) 281 goto err; 282 283 gve_rx_add_to_block(priv, idx); 284 285 return 0; 286 287 err: 288 gve_rx_free_ring_dqo(priv, idx); 289 return -ENOMEM; 290 } 291 292 void gve_rx_write_doorbell_dqo(const struct gve_priv *priv, int queue_idx) 293 { 294 const struct gve_rx_ring *rx = &priv->rx[queue_idx]; 295 u64 index = be32_to_cpu(rx->q_resources->db_index); 296 297 iowrite32(rx->dqo.bufq.tail, &priv->db_bar2[index]); 298 } 299 300 int gve_rx_alloc_rings_dqo(struct gve_priv *priv) 301 { 302 int err = 0; 303 int i; 304 305 for (i = 0; i < priv->rx_cfg.num_queues; i++) { 306 err = gve_rx_alloc_ring_dqo(priv, i); 307 if (err) { 308 netif_err(priv, drv, priv->dev, 309 "Failed to alloc rx ring=%d: err=%d\n", 310 i, err); 311 goto err; 312 } 313 } 314 315 return 0; 316 317 err: 318 for (i--; i >= 0; i--) 319 gve_rx_free_ring_dqo(priv, i); 320 321 return err; 322 } 323 324 void gve_rx_free_rings_dqo(struct gve_priv *priv) 325 { 326 int i; 327 328 for (i = 0; i < priv->rx_cfg.num_queues; i++) 329 gve_rx_free_ring_dqo(priv, i); 330 } 331 332 void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) 333 { 334 struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq; 335 struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq; 336 struct gve_priv *priv = rx->gve; 337 u32 num_avail_slots; 338 u32 num_full_slots; 339 u32 num_posted = 0; 340 341 num_full_slots = (bufq->tail - bufq->head) & bufq->mask; 342 num_avail_slots = bufq->mask - num_full_slots; 343 344 num_avail_slots = min_t(u32, num_avail_slots, complq->num_free_slots); 345 while (num_posted < num_avail_slots) { 346 struct gve_rx_desc_dqo *desc = &bufq->desc_ring[bufq->tail]; 347 struct gve_rx_buf_state_dqo *buf_state; 348 349 buf_state = gve_get_recycled_buf_state(rx); 350 if (unlikely(!buf_state)) { 351 buf_state = gve_alloc_buf_state(rx); 352 if (unlikely(!buf_state)) 353 break; 354 355 if (unlikely(gve_alloc_page_dqo(priv, buf_state))) { 356 u64_stats_update_begin(&rx->statss); 357 rx->rx_buf_alloc_fail++; 358 u64_stats_update_end(&rx->statss); 359 gve_free_buf_state(rx, buf_state); 360 break; 361 } 362 } 363 364 desc->buf_id = cpu_to_le16(buf_state - rx->dqo.buf_states); 365 desc->buf_addr = cpu_to_le64(buf_state->addr + 366 buf_state->page_info.page_offset); 367 368 bufq->tail = (bufq->tail + 1) & bufq->mask; 369 complq->num_free_slots--; 370 num_posted++; 371 372 if ((bufq->tail & (GVE_RX_BUF_THRESH_DQO - 1)) == 0) 373 gve_rx_write_doorbell_dqo(priv, rx->q_num); 374 } 375 376 rx->fill_cnt += num_posted; 377 } 378 379 static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx, 380 struct gve_rx_buf_state_dqo *buf_state) 381 { 382 const int data_buffer_size = priv->data_buffer_size_dqo; 383 int pagecount; 384 385 /* Can't reuse if we only fit one buffer per page */ 386 if (data_buffer_size * 2 > PAGE_SIZE) 387 goto mark_used; 388 389 pagecount = gve_buf_ref_cnt(buf_state); 390 391 /* Record the offset when we have a single remaining reference. 392 * 393 * When this happens, we know all of the other offsets of the page are 394 * usable. 395 */ 396 if (pagecount == 1) { 397 buf_state->last_single_ref_offset = 398 buf_state->page_info.page_offset; 399 } 400 401 /* Use the next buffer sized chunk in the page. */ 402 buf_state->page_info.page_offset += data_buffer_size; 403 buf_state->page_info.page_offset &= (PAGE_SIZE - 1); 404 405 /* If we wrap around to the same offset without ever dropping to 1 406 * reference, then we don't know if this offset was ever freed. 407 */ 408 if (buf_state->page_info.page_offset == 409 buf_state->last_single_ref_offset) { 410 goto mark_used; 411 } 412 413 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); 414 return; 415 416 mark_used: 417 gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state); 418 } 419 420 static void gve_rx_skb_csum(struct sk_buff *skb, 421 const struct gve_rx_compl_desc_dqo *desc, 422 struct gve_ptype ptype) 423 { 424 skb->ip_summed = CHECKSUM_NONE; 425 426 /* HW did not identify and process L3 and L4 headers. */ 427 if (unlikely(!desc->l3_l4_processed)) 428 return; 429 430 if (ptype.l3_type == GVE_L3_TYPE_IPV4) { 431 if (unlikely(desc->csum_ip_err || desc->csum_external_ip_err)) 432 return; 433 } else if (ptype.l3_type == GVE_L3_TYPE_IPV6) { 434 /* Checksum should be skipped if this flag is set. */ 435 if (unlikely(desc->ipv6_ex_add)) 436 return; 437 } 438 439 if (unlikely(desc->csum_l4_err)) 440 return; 441 442 switch (ptype.l4_type) { 443 case GVE_L4_TYPE_TCP: 444 case GVE_L4_TYPE_UDP: 445 case GVE_L4_TYPE_ICMP: 446 case GVE_L4_TYPE_SCTP: 447 skb->ip_summed = CHECKSUM_UNNECESSARY; 448 break; 449 default: 450 break; 451 } 452 } 453 454 static void gve_rx_skb_hash(struct sk_buff *skb, 455 const struct gve_rx_compl_desc_dqo *compl_desc, 456 struct gve_ptype ptype) 457 { 458 enum pkt_hash_types hash_type = PKT_HASH_TYPE_L2; 459 460 if (ptype.l4_type != GVE_L4_TYPE_UNKNOWN) 461 hash_type = PKT_HASH_TYPE_L4; 462 else if (ptype.l3_type != GVE_L3_TYPE_UNKNOWN) 463 hash_type = PKT_HASH_TYPE_L3; 464 465 skb_set_hash(skb, le32_to_cpu(compl_desc->hash), hash_type); 466 } 467 468 static void gve_rx_free_skb(struct gve_rx_ring *rx) 469 { 470 if (!rx->skb_head) 471 return; 472 473 dev_kfree_skb_any(rx->skb_head); 474 rx->skb_head = NULL; 475 rx->skb_tail = NULL; 476 } 477 478 /* Chains multi skbs for single rx packet. 479 * Returns 0 if buffer is appended, -1 otherwise. 480 */ 481 static int gve_rx_append_frags(struct napi_struct *napi, 482 struct gve_rx_buf_state_dqo *buf_state, 483 u16 buf_len, struct gve_rx_ring *rx, 484 struct gve_priv *priv) 485 { 486 int num_frags = skb_shinfo(rx->skb_tail)->nr_frags; 487 488 if (unlikely(num_frags == MAX_SKB_FRAGS)) { 489 struct sk_buff *skb; 490 491 skb = napi_alloc_skb(napi, 0); 492 if (!skb) 493 return -1; 494 495 skb_shinfo(rx->skb_tail)->frag_list = skb; 496 rx->skb_tail = skb; 497 num_frags = 0; 498 } 499 if (rx->skb_tail != rx->skb_head) { 500 rx->skb_head->len += buf_len; 501 rx->skb_head->data_len += buf_len; 502 rx->skb_head->truesize += priv->data_buffer_size_dqo; 503 } 504 505 skb_add_rx_frag(rx->skb_tail, num_frags, 506 buf_state->page_info.page, 507 buf_state->page_info.page_offset, 508 buf_len, priv->data_buffer_size_dqo); 509 gve_dec_pagecnt_bias(&buf_state->page_info); 510 511 return 0; 512 } 513 514 /* Returns 0 if descriptor is completed successfully. 515 * Returns -EINVAL if descriptor is invalid. 516 * Returns -ENOMEM if data cannot be copied to skb. 517 */ 518 static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, 519 const struct gve_rx_compl_desc_dqo *compl_desc, 520 int queue_idx) 521 { 522 const u16 buffer_id = le16_to_cpu(compl_desc->buf_id); 523 const bool eop = compl_desc->end_of_packet != 0; 524 struct gve_rx_buf_state_dqo *buf_state; 525 struct gve_priv *priv = rx->gve; 526 u16 buf_len; 527 528 if (unlikely(buffer_id >= rx->dqo.num_buf_states)) { 529 net_err_ratelimited("%s: Invalid RX buffer_id=%u\n", 530 priv->dev->name, buffer_id); 531 return -EINVAL; 532 } 533 buf_state = &rx->dqo.buf_states[buffer_id]; 534 if (unlikely(!gve_buf_state_is_allocated(rx, buf_state))) { 535 net_err_ratelimited("%s: RX buffer_id is not allocated: %u\n", 536 priv->dev->name, buffer_id); 537 return -EINVAL; 538 } 539 540 if (unlikely(compl_desc->rx_error)) { 541 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, 542 buf_state); 543 return -EINVAL; 544 } 545 546 buf_len = compl_desc->packet_len; 547 548 /* Page might have not been used for awhile and was likely last written 549 * by a different thread. 550 */ 551 prefetch(buf_state->page_info.page); 552 553 /* Sync the portion of dma buffer for CPU to read. */ 554 dma_sync_single_range_for_cpu(&priv->pdev->dev, buf_state->addr, 555 buf_state->page_info.page_offset, 556 buf_len, DMA_FROM_DEVICE); 557 558 /* Append to current skb if one exists. */ 559 if (rx->skb_head) { 560 if (unlikely(gve_rx_append_frags(napi, buf_state, buf_len, rx, 561 priv)) != 0) { 562 goto error; 563 } 564 565 gve_try_recycle_buf(priv, rx, buf_state); 566 return 0; 567 } 568 569 /* Prefetch the payload header. */ 570 prefetch((char *)buf_state->addr + buf_state->page_info.page_offset); 571 #if L1_CACHE_BYTES < 128 572 prefetch((char *)buf_state->addr + buf_state->page_info.page_offset + 573 L1_CACHE_BYTES); 574 #endif 575 576 if (eop && buf_len <= priv->rx_copybreak) { 577 rx->skb_head = gve_rx_copy(priv->dev, napi, 578 &buf_state->page_info, buf_len, 0); 579 if (unlikely(!rx->skb_head)) 580 goto error; 581 rx->skb_tail = rx->skb_head; 582 583 u64_stats_update_begin(&rx->statss); 584 rx->rx_copied_pkt++; 585 rx->rx_copybreak_pkt++; 586 u64_stats_update_end(&rx->statss); 587 588 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, 589 buf_state); 590 return 0; 591 } 592 593 rx->skb_head = napi_get_frags(napi); 594 if (unlikely(!rx->skb_head)) 595 goto error; 596 rx->skb_tail = rx->skb_head; 597 598 skb_add_rx_frag(rx->skb_head, 0, buf_state->page_info.page, 599 buf_state->page_info.page_offset, buf_len, 600 priv->data_buffer_size_dqo); 601 gve_dec_pagecnt_bias(&buf_state->page_info); 602 603 gve_try_recycle_buf(priv, rx, buf_state); 604 return 0; 605 606 error: 607 gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state); 608 return -ENOMEM; 609 } 610 611 static int gve_rx_complete_rsc(struct sk_buff *skb, 612 const struct gve_rx_compl_desc_dqo *desc, 613 struct gve_ptype ptype) 614 { 615 struct skb_shared_info *shinfo = skb_shinfo(skb); 616 617 /* Only TCP is supported right now. */ 618 if (ptype.l4_type != GVE_L4_TYPE_TCP) 619 return -EINVAL; 620 621 switch (ptype.l3_type) { 622 case GVE_L3_TYPE_IPV4: 623 shinfo->gso_type = SKB_GSO_TCPV4; 624 break; 625 case GVE_L3_TYPE_IPV6: 626 shinfo->gso_type = SKB_GSO_TCPV6; 627 break; 628 default: 629 return -EINVAL; 630 } 631 632 shinfo->gso_size = le16_to_cpu(desc->rsc_seg_len); 633 return 0; 634 } 635 636 /* Returns 0 if skb is completed successfully, -1 otherwise. */ 637 static int gve_rx_complete_skb(struct gve_rx_ring *rx, struct napi_struct *napi, 638 const struct gve_rx_compl_desc_dqo *desc, 639 netdev_features_t feat) 640 { 641 struct gve_ptype ptype = 642 rx->gve->ptype_lut_dqo->ptypes[desc->packet_type]; 643 int err; 644 645 skb_record_rx_queue(rx->skb_head, rx->q_num); 646 647 if (feat & NETIF_F_RXHASH) 648 gve_rx_skb_hash(rx->skb_head, desc, ptype); 649 650 if (feat & NETIF_F_RXCSUM) 651 gve_rx_skb_csum(rx->skb_head, desc, ptype); 652 653 /* RSC packets must set gso_size otherwise the TCP stack will complain 654 * that packets are larger than MTU. 655 */ 656 if (desc->rsc) { 657 err = gve_rx_complete_rsc(rx->skb_head, desc, ptype); 658 if (err < 0) 659 return err; 660 } 661 662 if (skb_headlen(rx->skb_head) == 0) 663 napi_gro_frags(napi); 664 else 665 napi_gro_receive(napi, rx->skb_head); 666 667 return 0; 668 } 669 670 int gve_rx_poll_dqo(struct gve_notify_block *block, int budget) 671 { 672 struct napi_struct *napi = &block->napi; 673 netdev_features_t feat = napi->dev->features; 674 675 struct gve_rx_ring *rx = block->rx; 676 struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq; 677 678 u32 work_done = 0; 679 u64 bytes = 0; 680 int err; 681 682 while (work_done < budget) { 683 struct gve_rx_compl_desc_dqo *compl_desc = 684 &complq->desc_ring[complq->head]; 685 u32 pkt_bytes; 686 687 /* No more new packets */ 688 if (compl_desc->generation == complq->cur_gen_bit) 689 break; 690 691 /* Prefetch the next two descriptors. */ 692 prefetch(&complq->desc_ring[(complq->head + 1) & complq->mask]); 693 prefetch(&complq->desc_ring[(complq->head + 2) & complq->mask]); 694 695 /* Do not read data until we own the descriptor */ 696 dma_rmb(); 697 698 err = gve_rx_dqo(napi, rx, compl_desc, rx->q_num); 699 if (err < 0) { 700 gve_rx_free_skb(rx); 701 u64_stats_update_begin(&rx->statss); 702 if (err == -ENOMEM) 703 rx->rx_skb_alloc_fail++; 704 else if (err == -EINVAL) 705 rx->rx_desc_err_dropped_pkt++; 706 u64_stats_update_end(&rx->statss); 707 } 708 709 complq->head = (complq->head + 1) & complq->mask; 710 complq->num_free_slots++; 711 712 /* When the ring wraps, the generation bit is flipped. */ 713 complq->cur_gen_bit ^= (complq->head == 0); 714 715 /* Receiving a completion means we have space to post another 716 * buffer on the buffer queue. 717 */ 718 { 719 struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq; 720 721 bufq->head = (bufq->head + 1) & bufq->mask; 722 } 723 724 /* Free running counter of completed descriptors */ 725 rx->cnt++; 726 727 if (!rx->skb_head) 728 continue; 729 730 if (!compl_desc->end_of_packet) 731 continue; 732 733 work_done++; 734 pkt_bytes = rx->skb_head->len; 735 /* The ethernet header (first ETH_HLEN bytes) is snipped off 736 * by eth_type_trans. 737 */ 738 if (skb_headlen(rx->skb_head)) 739 pkt_bytes += ETH_HLEN; 740 741 /* gve_rx_complete_skb() will consume skb if successful */ 742 if (gve_rx_complete_skb(rx, napi, compl_desc, feat) != 0) { 743 gve_rx_free_skb(rx); 744 u64_stats_update_begin(&rx->statss); 745 rx->rx_desc_err_dropped_pkt++; 746 u64_stats_update_end(&rx->statss); 747 continue; 748 } 749 750 bytes += pkt_bytes; 751 rx->skb_head = NULL; 752 rx->skb_tail = NULL; 753 } 754 755 gve_rx_post_buffers_dqo(rx); 756 757 u64_stats_update_begin(&rx->statss); 758 rx->rpackets += work_done; 759 rx->rbytes += bytes; 760 u64_stats_update_end(&rx->statss); 761 762 return work_done; 763 } 764