1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include "gve.h" 8 #include "gve_adminq.h" 9 #include "gve_utils.h" 10 #include "gve_dqo.h" 11 #include <net/ip.h> 12 #include <linux/tcp.h> 13 #include <linux/slab.h> 14 #include <linux/skbuff.h> 15 16 /* Returns true if tx_bufs are available. */ 17 static bool gve_has_free_tx_qpl_bufs(struct gve_tx_ring *tx, int count) 18 { 19 int num_avail; 20 21 if (!tx->dqo.qpl) 22 return true; 23 24 num_avail = tx->dqo.num_tx_qpl_bufs - 25 (tx->dqo_tx.alloc_tx_qpl_buf_cnt - 26 tx->dqo_tx.free_tx_qpl_buf_cnt); 27 28 if (count <= num_avail) 29 return true; 30 31 /* Update cached value from dqo_compl. */ 32 tx->dqo_tx.free_tx_qpl_buf_cnt = 33 atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_cnt); 34 35 num_avail = tx->dqo.num_tx_qpl_bufs - 36 (tx->dqo_tx.alloc_tx_qpl_buf_cnt - 37 tx->dqo_tx.free_tx_qpl_buf_cnt); 38 39 return count <= num_avail; 40 } 41 42 static s16 43 gve_alloc_tx_qpl_buf(struct gve_tx_ring *tx) 44 { 45 s16 index; 46 47 index = tx->dqo_tx.free_tx_qpl_buf_head; 48 49 /* No TX buffers available, try to steal the list from the 50 * completion handler. 51 */ 52 if (unlikely(index == -1)) { 53 tx->dqo_tx.free_tx_qpl_buf_head = 54 atomic_xchg(&tx->dqo_compl.free_tx_qpl_buf_head, -1); 55 index = tx->dqo_tx.free_tx_qpl_buf_head; 56 57 if (unlikely(index == -1)) 58 return index; 59 } 60 61 /* Remove TX buf from free list */ 62 tx->dqo_tx.free_tx_qpl_buf_head = tx->dqo.tx_qpl_buf_next[index]; 63 64 return index; 65 } 66 67 static void 68 gve_free_tx_qpl_bufs(struct gve_tx_ring *tx, 69 struct gve_tx_pending_packet_dqo *pkt) 70 { 71 s16 index; 72 int i; 73 74 if (!pkt->num_bufs) 75 return; 76 77 index = pkt->tx_qpl_buf_ids[0]; 78 /* Create a linked list of buffers to be added to the free list */ 79 for (i = 1; i < pkt->num_bufs; i++) { 80 tx->dqo.tx_qpl_buf_next[index] = pkt->tx_qpl_buf_ids[i]; 81 index = pkt->tx_qpl_buf_ids[i]; 82 } 83 84 while (true) { 85 s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_tx_qpl_buf_head); 86 87 tx->dqo.tx_qpl_buf_next[index] = old_head; 88 if (atomic_cmpxchg(&tx->dqo_compl.free_tx_qpl_buf_head, 89 old_head, 90 pkt->tx_qpl_buf_ids[0]) == old_head) { 91 break; 92 } 93 } 94 95 atomic_add(pkt->num_bufs, &tx->dqo_compl.free_tx_qpl_buf_cnt); 96 pkt->num_bufs = 0; 97 } 98 99 /* Returns true if a gve_tx_pending_packet_dqo object is available. */ 100 static bool gve_has_pending_packet(struct gve_tx_ring *tx) 101 { 102 /* Check TX path's list. */ 103 if (tx->dqo_tx.free_pending_packets != -1) 104 return true; 105 106 /* Check completion handler's list. */ 107 if (atomic_read_acquire(&tx->dqo_compl.free_pending_packets) != -1) 108 return true; 109 110 return false; 111 } 112 113 static struct gve_tx_pending_packet_dqo * 114 gve_alloc_pending_packet(struct gve_tx_ring *tx) 115 { 116 struct gve_tx_pending_packet_dqo *pending_packet; 117 s16 index; 118 119 index = tx->dqo_tx.free_pending_packets; 120 121 /* No pending_packets available, try to steal the list from the 122 * completion handler. 123 */ 124 if (unlikely(index == -1)) { 125 tx->dqo_tx.free_pending_packets = 126 atomic_xchg(&tx->dqo_compl.free_pending_packets, -1); 127 index = tx->dqo_tx.free_pending_packets; 128 129 if (unlikely(index == -1)) 130 return NULL; 131 } 132 133 pending_packet = &tx->dqo.pending_packets[index]; 134 135 /* Remove pending_packet from free list */ 136 tx->dqo_tx.free_pending_packets = pending_packet->next; 137 pending_packet->state = GVE_PACKET_STATE_PENDING_DATA_COMPL; 138 139 return pending_packet; 140 } 141 142 static void 143 gve_free_pending_packet(struct gve_tx_ring *tx, 144 struct gve_tx_pending_packet_dqo *pending_packet) 145 { 146 s16 index = pending_packet - tx->dqo.pending_packets; 147 148 pending_packet->state = GVE_PACKET_STATE_UNALLOCATED; 149 while (true) { 150 s16 old_head = atomic_read_acquire(&tx->dqo_compl.free_pending_packets); 151 152 pending_packet->next = old_head; 153 if (atomic_cmpxchg(&tx->dqo_compl.free_pending_packets, 154 old_head, index) == old_head) { 155 break; 156 } 157 } 158 } 159 160 /* gve_tx_free_desc - Cleans up all pending tx requests and buffers. 161 */ 162 static void gve_tx_clean_pending_packets(struct gve_tx_ring *tx) 163 { 164 int i; 165 166 for (i = 0; i < tx->dqo.num_pending_packets; i++) { 167 struct gve_tx_pending_packet_dqo *cur_state = 168 &tx->dqo.pending_packets[i]; 169 int j; 170 171 for (j = 0; j < cur_state->num_bufs; j++) { 172 if (j == 0) { 173 dma_unmap_single(tx->dev, 174 dma_unmap_addr(cur_state, dma[j]), 175 dma_unmap_len(cur_state, len[j]), 176 DMA_TO_DEVICE); 177 } else { 178 dma_unmap_page(tx->dev, 179 dma_unmap_addr(cur_state, dma[j]), 180 dma_unmap_len(cur_state, len[j]), 181 DMA_TO_DEVICE); 182 } 183 } 184 if (cur_state->skb) { 185 dev_consume_skb_any(cur_state->skb); 186 cur_state->skb = NULL; 187 } 188 } 189 } 190 191 static void gve_tx_free_ring_dqo(struct gve_priv *priv, int idx) 192 { 193 struct gve_tx_ring *tx = &priv->tx[idx]; 194 struct device *hdev = &priv->pdev->dev; 195 size_t bytes; 196 197 gve_tx_remove_from_block(priv, idx); 198 199 if (tx->q_resources) { 200 dma_free_coherent(hdev, sizeof(*tx->q_resources), 201 tx->q_resources, tx->q_resources_bus); 202 tx->q_resources = NULL; 203 } 204 205 if (tx->dqo.compl_ring) { 206 bytes = sizeof(tx->dqo.compl_ring[0]) * 207 (tx->dqo.complq_mask + 1); 208 dma_free_coherent(hdev, bytes, tx->dqo.compl_ring, 209 tx->complq_bus_dqo); 210 tx->dqo.compl_ring = NULL; 211 } 212 213 if (tx->dqo.tx_ring) { 214 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); 215 dma_free_coherent(hdev, bytes, tx->dqo.tx_ring, tx->bus); 216 tx->dqo.tx_ring = NULL; 217 } 218 219 kvfree(tx->dqo.pending_packets); 220 tx->dqo.pending_packets = NULL; 221 222 kvfree(tx->dqo.tx_qpl_buf_next); 223 tx->dqo.tx_qpl_buf_next = NULL; 224 225 if (tx->dqo.qpl) { 226 gve_unassign_qpl(priv, tx->dqo.qpl->id); 227 tx->dqo.qpl = NULL; 228 } 229 230 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); 231 } 232 233 static int gve_tx_qpl_buf_init(struct gve_tx_ring *tx) 234 { 235 int num_tx_qpl_bufs = GVE_TX_BUFS_PER_PAGE_DQO * 236 tx->dqo.qpl->num_entries; 237 int i; 238 239 tx->dqo.tx_qpl_buf_next = kvcalloc(num_tx_qpl_bufs, 240 sizeof(tx->dqo.tx_qpl_buf_next[0]), 241 GFP_KERNEL); 242 if (!tx->dqo.tx_qpl_buf_next) 243 return -ENOMEM; 244 245 tx->dqo.num_tx_qpl_bufs = num_tx_qpl_bufs; 246 247 /* Generate free TX buf list */ 248 for (i = 0; i < num_tx_qpl_bufs - 1; i++) 249 tx->dqo.tx_qpl_buf_next[i] = i + 1; 250 tx->dqo.tx_qpl_buf_next[num_tx_qpl_bufs - 1] = -1; 251 252 atomic_set_release(&tx->dqo_compl.free_tx_qpl_buf_head, -1); 253 return 0; 254 } 255 256 static int gve_tx_alloc_ring_dqo(struct gve_priv *priv, int idx) 257 { 258 struct gve_tx_ring *tx = &priv->tx[idx]; 259 struct device *hdev = &priv->pdev->dev; 260 int num_pending_packets; 261 size_t bytes; 262 int i; 263 264 memset(tx, 0, sizeof(*tx)); 265 tx->q_num = idx; 266 tx->dev = &priv->pdev->dev; 267 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx); 268 atomic_set_release(&tx->dqo_compl.hw_tx_head, 0); 269 270 /* Queue sizes must be a power of 2 */ 271 tx->mask = priv->tx_desc_cnt - 1; 272 tx->dqo.complq_mask = priv->queue_format == GVE_DQO_RDA_FORMAT ? 273 priv->options_dqo_rda.tx_comp_ring_entries - 1 : 274 tx->mask; 275 276 /* The max number of pending packets determines the maximum number of 277 * descriptors which maybe written to the completion queue. 278 * 279 * We must set the number small enough to make sure we never overrun the 280 * completion queue. 281 */ 282 num_pending_packets = tx->dqo.complq_mask + 1; 283 284 /* Reserve space for descriptor completions, which will be reported at 285 * most every GVE_TX_MIN_RE_INTERVAL packets. 286 */ 287 num_pending_packets -= 288 (tx->dqo.complq_mask + 1) / GVE_TX_MIN_RE_INTERVAL; 289 290 /* Each packet may have at most 2 buffer completions if it receives both 291 * a miss and reinjection completion. 292 */ 293 num_pending_packets /= 2; 294 295 tx->dqo.num_pending_packets = min_t(int, num_pending_packets, S16_MAX); 296 tx->dqo.pending_packets = kvcalloc(tx->dqo.num_pending_packets, 297 sizeof(tx->dqo.pending_packets[0]), 298 GFP_KERNEL); 299 if (!tx->dqo.pending_packets) 300 goto err; 301 302 /* Set up linked list of pending packets */ 303 for (i = 0; i < tx->dqo.num_pending_packets - 1; i++) 304 tx->dqo.pending_packets[i].next = i + 1; 305 306 tx->dqo.pending_packets[tx->dqo.num_pending_packets - 1].next = -1; 307 atomic_set_release(&tx->dqo_compl.free_pending_packets, -1); 308 tx->dqo_compl.miss_completions.head = -1; 309 tx->dqo_compl.miss_completions.tail = -1; 310 tx->dqo_compl.timed_out_completions.head = -1; 311 tx->dqo_compl.timed_out_completions.tail = -1; 312 313 bytes = sizeof(tx->dqo.tx_ring[0]) * (tx->mask + 1); 314 tx->dqo.tx_ring = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL); 315 if (!tx->dqo.tx_ring) 316 goto err; 317 318 bytes = sizeof(tx->dqo.compl_ring[0]) * (tx->dqo.complq_mask + 1); 319 tx->dqo.compl_ring = dma_alloc_coherent(hdev, bytes, 320 &tx->complq_bus_dqo, 321 GFP_KERNEL); 322 if (!tx->dqo.compl_ring) 323 goto err; 324 325 tx->q_resources = dma_alloc_coherent(hdev, sizeof(*tx->q_resources), 326 &tx->q_resources_bus, GFP_KERNEL); 327 if (!tx->q_resources) 328 goto err; 329 330 if (gve_is_qpl(priv)) { 331 tx->dqo.qpl = gve_assign_tx_qpl(priv, idx); 332 if (!tx->dqo.qpl) 333 goto err; 334 335 if (gve_tx_qpl_buf_init(tx)) 336 goto err; 337 } 338 339 gve_tx_add_to_block(priv, idx); 340 341 return 0; 342 343 err: 344 gve_tx_free_ring_dqo(priv, idx); 345 return -ENOMEM; 346 } 347 348 int gve_tx_alloc_rings_dqo(struct gve_priv *priv) 349 { 350 int err = 0; 351 int i; 352 353 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 354 err = gve_tx_alloc_ring_dqo(priv, i); 355 if (err) { 356 netif_err(priv, drv, priv->dev, 357 "Failed to alloc tx ring=%d: err=%d\n", 358 i, err); 359 goto err; 360 } 361 } 362 363 return 0; 364 365 err: 366 for (i--; i >= 0; i--) 367 gve_tx_free_ring_dqo(priv, i); 368 369 return err; 370 } 371 372 void gve_tx_free_rings_dqo(struct gve_priv *priv) 373 { 374 int i; 375 376 for (i = 0; i < priv->tx_cfg.num_queues; i++) { 377 struct gve_tx_ring *tx = &priv->tx[i]; 378 379 gve_clean_tx_done_dqo(priv, tx, /*napi=*/NULL); 380 netdev_tx_reset_queue(tx->netdev_txq); 381 gve_tx_clean_pending_packets(tx); 382 383 gve_tx_free_ring_dqo(priv, i); 384 } 385 } 386 387 /* Returns the number of slots available in the ring */ 388 static u32 num_avail_tx_slots(const struct gve_tx_ring *tx) 389 { 390 u32 num_used = (tx->dqo_tx.tail - tx->dqo_tx.head) & tx->mask; 391 392 return tx->mask - num_used; 393 } 394 395 static bool gve_has_avail_slots_tx_dqo(struct gve_tx_ring *tx, 396 int desc_count, int buf_count) 397 { 398 return gve_has_pending_packet(tx) && 399 num_avail_tx_slots(tx) >= desc_count && 400 gve_has_free_tx_qpl_bufs(tx, buf_count); 401 } 402 403 /* Stops the queue if available descriptors is less than 'count'. 404 * Return: 0 if stop is not required. 405 */ 406 static int gve_maybe_stop_tx_dqo(struct gve_tx_ring *tx, 407 int desc_count, int buf_count) 408 { 409 if (likely(gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count))) 410 return 0; 411 412 /* Update cached TX head pointer */ 413 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); 414 415 if (likely(gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count))) 416 return 0; 417 418 /* No space, so stop the queue */ 419 tx->stop_queue++; 420 netif_tx_stop_queue(tx->netdev_txq); 421 422 /* Sync with restarting queue in `gve_tx_poll_dqo()` */ 423 mb(); 424 425 /* After stopping queue, check if we can transmit again in order to 426 * avoid TOCTOU bug. 427 */ 428 tx->dqo_tx.head = atomic_read_acquire(&tx->dqo_compl.hw_tx_head); 429 430 if (likely(!gve_has_avail_slots_tx_dqo(tx, desc_count, buf_count))) 431 return -EBUSY; 432 433 netif_tx_start_queue(tx->netdev_txq); 434 tx->wake_queue++; 435 return 0; 436 } 437 438 static void gve_extract_tx_metadata_dqo(const struct sk_buff *skb, 439 struct gve_tx_metadata_dqo *metadata) 440 { 441 memset(metadata, 0, sizeof(*metadata)); 442 metadata->version = GVE_TX_METADATA_VERSION_DQO; 443 444 if (skb->l4_hash) { 445 u16 path_hash = skb->hash ^ (skb->hash >> 16); 446 447 path_hash &= (1 << 15) - 1; 448 if (unlikely(path_hash == 0)) 449 path_hash = ~path_hash; 450 451 metadata->path_hash = path_hash; 452 } 453 } 454 455 static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, 456 struct sk_buff *skb, u32 len, u64 addr, 457 s16 compl_tag, bool eop, bool is_gso) 458 { 459 const bool checksum_offload_en = skb->ip_summed == CHECKSUM_PARTIAL; 460 461 while (len > 0) { 462 struct gve_tx_pkt_desc_dqo *desc = 463 &tx->dqo.tx_ring[*desc_idx].pkt; 464 u32 cur_len = min_t(u32, len, GVE_TX_MAX_BUF_SIZE_DQO); 465 bool cur_eop = eop && cur_len == len; 466 467 *desc = (struct gve_tx_pkt_desc_dqo){ 468 .buf_addr = cpu_to_le64(addr), 469 .dtype = GVE_TX_PKT_DESC_DTYPE_DQO, 470 .end_of_packet = cur_eop, 471 .checksum_offload_enable = checksum_offload_en, 472 .compl_tag = cpu_to_le16(compl_tag), 473 .buf_size = cur_len, 474 }; 475 476 addr += cur_len; 477 len -= cur_len; 478 *desc_idx = (*desc_idx + 1) & tx->mask; 479 } 480 } 481 482 /* Validates and prepares `skb` for TSO. 483 * 484 * Returns header length, or < 0 if invalid. 485 */ 486 static int gve_prep_tso(struct sk_buff *skb) 487 { 488 struct tcphdr *tcp; 489 int header_len; 490 u32 paylen; 491 int err; 492 493 /* Note: HW requires MSS (gso_size) to be <= 9728 and the total length 494 * of the TSO to be <= 262143. 495 * 496 * However, we don't validate these because: 497 * - Hypervisor enforces a limit of 9K MTU 498 * - Kernel will not produce a TSO larger than 64k 499 */ 500 501 if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) 502 return -1; 503 504 /* Needed because we will modify header. */ 505 err = skb_cow_head(skb, 0); 506 if (err < 0) 507 return err; 508 509 tcp = tcp_hdr(skb); 510 511 /* Remove payload length from checksum. */ 512 paylen = skb->len - skb_transport_offset(skb); 513 514 switch (skb_shinfo(skb)->gso_type) { 515 case SKB_GSO_TCPV4: 516 case SKB_GSO_TCPV6: 517 csum_replace_by_diff(&tcp->check, 518 (__force __wsum)htonl(paylen)); 519 520 /* Compute length of segmentation header. */ 521 header_len = skb_tcp_all_headers(skb); 522 break; 523 default: 524 return -EINVAL; 525 } 526 527 if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO)) 528 return -EINVAL; 529 530 return header_len; 531 } 532 533 static void gve_tx_fill_tso_ctx_desc(struct gve_tx_tso_context_desc_dqo *desc, 534 const struct sk_buff *skb, 535 const struct gve_tx_metadata_dqo *metadata, 536 int header_len) 537 { 538 *desc = (struct gve_tx_tso_context_desc_dqo){ 539 .header_len = header_len, 540 .cmd_dtype = { 541 .dtype = GVE_TX_TSO_CTX_DESC_DTYPE_DQO, 542 .tso = 1, 543 }, 544 .flex0 = metadata->bytes[0], 545 .flex5 = metadata->bytes[5], 546 .flex6 = metadata->bytes[6], 547 .flex7 = metadata->bytes[7], 548 .flex8 = metadata->bytes[8], 549 .flex9 = metadata->bytes[9], 550 .flex10 = metadata->bytes[10], 551 .flex11 = metadata->bytes[11], 552 }; 553 desc->tso_total_len = skb->len - header_len; 554 desc->mss = skb_shinfo(skb)->gso_size; 555 } 556 557 static void 558 gve_tx_fill_general_ctx_desc(struct gve_tx_general_context_desc_dqo *desc, 559 const struct gve_tx_metadata_dqo *metadata) 560 { 561 *desc = (struct gve_tx_general_context_desc_dqo){ 562 .flex0 = metadata->bytes[0], 563 .flex1 = metadata->bytes[1], 564 .flex2 = metadata->bytes[2], 565 .flex3 = metadata->bytes[3], 566 .flex4 = metadata->bytes[4], 567 .flex5 = metadata->bytes[5], 568 .flex6 = metadata->bytes[6], 569 .flex7 = metadata->bytes[7], 570 .flex8 = metadata->bytes[8], 571 .flex9 = metadata->bytes[9], 572 .flex10 = metadata->bytes[10], 573 .flex11 = metadata->bytes[11], 574 .cmd_dtype = {.dtype = GVE_TX_GENERAL_CTX_DESC_DTYPE_DQO}, 575 }; 576 } 577 578 static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx, 579 struct sk_buff *skb, 580 struct gve_tx_pending_packet_dqo *pkt, 581 s16 completion_tag, 582 u32 *desc_idx, 583 bool is_gso) 584 { 585 const struct skb_shared_info *shinfo = skb_shinfo(skb); 586 int i; 587 588 /* Note: HW requires that the size of a non-TSO packet be within the 589 * range of [17, 9728]. 590 * 591 * We don't double check because 592 * - We limited `netdev->min_mtu` to ETH_MIN_MTU. 593 * - Hypervisor won't allow MTU larger than 9216. 594 */ 595 596 pkt->num_bufs = 0; 597 /* Map the linear portion of skb */ 598 { 599 u32 len = skb_headlen(skb); 600 dma_addr_t addr; 601 602 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE); 603 if (unlikely(dma_mapping_error(tx->dev, addr))) 604 goto err; 605 606 dma_unmap_len_set(pkt, len[pkt->num_bufs], len); 607 dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr); 608 ++pkt->num_bufs; 609 610 gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, len, addr, 611 completion_tag, 612 /*eop=*/shinfo->nr_frags == 0, is_gso); 613 } 614 615 for (i = 0; i < shinfo->nr_frags; i++) { 616 const skb_frag_t *frag = &shinfo->frags[i]; 617 bool is_eop = i == (shinfo->nr_frags - 1); 618 u32 len = skb_frag_size(frag); 619 dma_addr_t addr; 620 621 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE); 622 if (unlikely(dma_mapping_error(tx->dev, addr))) 623 goto err; 624 625 dma_unmap_len_set(pkt, len[pkt->num_bufs], len); 626 dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr); 627 ++pkt->num_bufs; 628 629 gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, len, addr, 630 completion_tag, is_eop, is_gso); 631 } 632 633 return 0; 634 err: 635 for (i = 0; i < pkt->num_bufs; i++) { 636 if (i == 0) { 637 dma_unmap_single(tx->dev, 638 dma_unmap_addr(pkt, dma[i]), 639 dma_unmap_len(pkt, len[i]), 640 DMA_TO_DEVICE); 641 } else { 642 dma_unmap_page(tx->dev, 643 dma_unmap_addr(pkt, dma[i]), 644 dma_unmap_len(pkt, len[i]), 645 DMA_TO_DEVICE); 646 } 647 } 648 pkt->num_bufs = 0; 649 return -1; 650 } 651 652 /* Tx buffer i corresponds to 653 * qpl_page_id = i / GVE_TX_BUFS_PER_PAGE_DQO 654 * qpl_page_offset = (i % GVE_TX_BUFS_PER_PAGE_DQO) * GVE_TX_BUF_SIZE_DQO 655 */ 656 static void gve_tx_buf_get_addr(struct gve_tx_ring *tx, 657 s16 index, 658 void **va, dma_addr_t *dma_addr) 659 { 660 int page_id = index >> (PAGE_SHIFT - GVE_TX_BUF_SHIFT_DQO); 661 int offset = (index & (GVE_TX_BUFS_PER_PAGE_DQO - 1)) << GVE_TX_BUF_SHIFT_DQO; 662 663 *va = page_address(tx->dqo.qpl->pages[page_id]) + offset; 664 *dma_addr = tx->dqo.qpl->page_buses[page_id] + offset; 665 } 666 667 static int gve_tx_add_skb_copy_dqo(struct gve_tx_ring *tx, 668 struct sk_buff *skb, 669 struct gve_tx_pending_packet_dqo *pkt, 670 s16 completion_tag, 671 u32 *desc_idx, 672 bool is_gso) 673 { 674 u32 copy_offset = 0; 675 dma_addr_t dma_addr; 676 u32 copy_len; 677 s16 index; 678 void *va; 679 680 /* Break the packet into buffer size chunks */ 681 pkt->num_bufs = 0; 682 while (copy_offset < skb->len) { 683 index = gve_alloc_tx_qpl_buf(tx); 684 if (unlikely(index == -1)) 685 goto err; 686 687 gve_tx_buf_get_addr(tx, index, &va, &dma_addr); 688 copy_len = min_t(u32, GVE_TX_BUF_SIZE_DQO, 689 skb->len - copy_offset); 690 skb_copy_bits(skb, copy_offset, va, copy_len); 691 692 copy_offset += copy_len; 693 dma_sync_single_for_device(tx->dev, dma_addr, 694 copy_len, DMA_TO_DEVICE); 695 gve_tx_fill_pkt_desc_dqo(tx, desc_idx, skb, 696 copy_len, 697 dma_addr, 698 completion_tag, 699 copy_offset == skb->len, 700 is_gso); 701 702 pkt->tx_qpl_buf_ids[pkt->num_bufs] = index; 703 ++tx->dqo_tx.alloc_tx_qpl_buf_cnt; 704 ++pkt->num_bufs; 705 } 706 707 return 0; 708 err: 709 /* Should not be here if gve_has_free_tx_qpl_bufs() check is correct */ 710 gve_free_tx_qpl_bufs(tx, pkt); 711 return -ENOMEM; 712 } 713 714 /* Returns 0 on success, or < 0 on error. 715 * 716 * Before this function is called, the caller must ensure 717 * gve_has_pending_packet(tx) returns true. 718 */ 719 static int gve_tx_add_skb_dqo(struct gve_tx_ring *tx, 720 struct sk_buff *skb) 721 { 722 const bool is_gso = skb_is_gso(skb); 723 u32 desc_idx = tx->dqo_tx.tail; 724 struct gve_tx_pending_packet_dqo *pkt; 725 struct gve_tx_metadata_dqo metadata; 726 s16 completion_tag; 727 728 pkt = gve_alloc_pending_packet(tx); 729 pkt->skb = skb; 730 completion_tag = pkt - tx->dqo.pending_packets; 731 732 gve_extract_tx_metadata_dqo(skb, &metadata); 733 if (is_gso) { 734 int header_len = gve_prep_tso(skb); 735 736 if (unlikely(header_len < 0)) 737 goto err; 738 739 gve_tx_fill_tso_ctx_desc(&tx->dqo.tx_ring[desc_idx].tso_ctx, 740 skb, &metadata, header_len); 741 desc_idx = (desc_idx + 1) & tx->mask; 742 } 743 744 gve_tx_fill_general_ctx_desc(&tx->dqo.tx_ring[desc_idx].general_ctx, 745 &metadata); 746 desc_idx = (desc_idx + 1) & tx->mask; 747 748 if (tx->dqo.qpl) { 749 if (gve_tx_add_skb_copy_dqo(tx, skb, pkt, 750 completion_tag, 751 &desc_idx, is_gso)) 752 goto err; 753 } else { 754 if (gve_tx_add_skb_no_copy_dqo(tx, skb, pkt, 755 completion_tag, 756 &desc_idx, is_gso)) 757 goto err; 758 } 759 760 tx->dqo_tx.posted_packet_desc_cnt += pkt->num_bufs; 761 762 /* Commit the changes to our state */ 763 tx->dqo_tx.tail = desc_idx; 764 765 /* Request a descriptor completion on the last descriptor of the 766 * packet if we are allowed to by the HW enforced interval. 767 */ 768 { 769 u32 last_desc_idx = (desc_idx - 1) & tx->mask; 770 u32 last_report_event_interval = 771 (last_desc_idx - tx->dqo_tx.last_re_idx) & tx->mask; 772 773 if (unlikely(last_report_event_interval >= 774 GVE_TX_MIN_RE_INTERVAL)) { 775 tx->dqo.tx_ring[last_desc_idx].pkt.report_event = true; 776 tx->dqo_tx.last_re_idx = last_desc_idx; 777 } 778 } 779 780 return 0; 781 782 err: 783 pkt->skb = NULL; 784 gve_free_pending_packet(tx, pkt); 785 786 return -1; 787 } 788 789 static int gve_num_descs_per_buf(size_t size) 790 { 791 return DIV_ROUND_UP(size, GVE_TX_MAX_BUF_SIZE_DQO); 792 } 793 794 static int gve_num_buffer_descs_needed(const struct sk_buff *skb) 795 { 796 const struct skb_shared_info *shinfo = skb_shinfo(skb); 797 int num_descs; 798 int i; 799 800 num_descs = gve_num_descs_per_buf(skb_headlen(skb)); 801 802 for (i = 0; i < shinfo->nr_frags; i++) { 803 unsigned int frag_size = skb_frag_size(&shinfo->frags[i]); 804 805 num_descs += gve_num_descs_per_buf(frag_size); 806 } 807 808 return num_descs; 809 } 810 811 /* Returns true if HW is capable of sending TSO represented by `skb`. 812 * 813 * Each segment must not span more than GVE_TX_MAX_DATA_DESCS buffers. 814 * - The header is counted as one buffer for every single segment. 815 * - A buffer which is split between two segments is counted for both. 816 * - If a buffer contains both header and payload, it is counted as two buffers. 817 */ 818 static bool gve_can_send_tso(const struct sk_buff *skb) 819 { 820 const int max_bufs_per_seg = GVE_TX_MAX_DATA_DESCS - 1; 821 const struct skb_shared_info *shinfo = skb_shinfo(skb); 822 const int header_len = skb_tcp_all_headers(skb); 823 const int gso_size = shinfo->gso_size; 824 int cur_seg_num_bufs; 825 int cur_seg_size; 826 int i; 827 828 cur_seg_size = skb_headlen(skb) - header_len; 829 cur_seg_num_bufs = cur_seg_size > 0; 830 831 for (i = 0; i < shinfo->nr_frags; i++) { 832 if (cur_seg_size >= gso_size) { 833 cur_seg_size %= gso_size; 834 cur_seg_num_bufs = cur_seg_size > 0; 835 } 836 837 if (unlikely(++cur_seg_num_bufs > max_bufs_per_seg)) 838 return false; 839 840 cur_seg_size += skb_frag_size(&shinfo->frags[i]); 841 } 842 843 return true; 844 } 845 846 /* Attempt to transmit specified SKB. 847 * 848 * Returns 0 if the SKB was transmitted or dropped. 849 * Returns -1 if there is not currently enough space to transmit the SKB. 850 */ 851 static int gve_try_tx_skb(struct gve_priv *priv, struct gve_tx_ring *tx, 852 struct sk_buff *skb) 853 { 854 int num_buffer_descs; 855 int total_num_descs; 856 857 if (tx->dqo.qpl) { 858 if (skb_is_gso(skb)) 859 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 860 goto drop; 861 862 /* We do not need to verify the number of buffers used per 863 * packet or per segment in case of TSO as with 2K size buffers 864 * none of the TX packet rules would be violated. 865 * 866 * gve_can_send_tso() checks that each TCP segment of gso_size is 867 * not distributed over more than 9 SKB frags.. 868 */ 869 num_buffer_descs = DIV_ROUND_UP(skb->len, GVE_TX_BUF_SIZE_DQO); 870 } else { 871 if (skb_is_gso(skb)) { 872 /* If TSO doesn't meet HW requirements, attempt to linearize the 873 * packet. 874 */ 875 if (unlikely(!gve_can_send_tso(skb) && 876 skb_linearize(skb) < 0)) { 877 net_err_ratelimited("%s: Failed to transmit TSO packet\n", 878 priv->dev->name); 879 goto drop; 880 } 881 882 if (unlikely(ipv6_hopopt_jumbo_remove(skb))) 883 goto drop; 884 885 num_buffer_descs = gve_num_buffer_descs_needed(skb); 886 } else { 887 num_buffer_descs = gve_num_buffer_descs_needed(skb); 888 889 if (unlikely(num_buffer_descs > GVE_TX_MAX_DATA_DESCS)) { 890 if (unlikely(skb_linearize(skb) < 0)) 891 goto drop; 892 893 num_buffer_descs = 1; 894 } 895 } 896 } 897 898 /* Metadata + (optional TSO) + data descriptors. */ 899 total_num_descs = 1 + skb_is_gso(skb) + num_buffer_descs; 900 if (unlikely(gve_maybe_stop_tx_dqo(tx, total_num_descs + 901 GVE_TX_MIN_DESC_PREVENT_CACHE_OVERLAP, 902 num_buffer_descs))) { 903 return -1; 904 } 905 906 if (unlikely(gve_tx_add_skb_dqo(tx, skb) < 0)) 907 goto drop; 908 909 netdev_tx_sent_queue(tx->netdev_txq, skb->len); 910 skb_tx_timestamp(skb); 911 return 0; 912 913 drop: 914 tx->dropped_pkt++; 915 dev_kfree_skb_any(skb); 916 return 0; 917 } 918 919 /* Transmit a given skb and ring the doorbell. */ 920 netdev_tx_t gve_tx_dqo(struct sk_buff *skb, struct net_device *dev) 921 { 922 struct gve_priv *priv = netdev_priv(dev); 923 struct gve_tx_ring *tx; 924 925 tx = &priv->tx[skb_get_queue_mapping(skb)]; 926 if (unlikely(gve_try_tx_skb(priv, tx, skb) < 0)) { 927 /* We need to ring the txq doorbell -- we have stopped the Tx 928 * queue for want of resources, but prior calls to gve_tx() 929 * may have added descriptors without ringing the doorbell. 930 */ 931 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); 932 return NETDEV_TX_BUSY; 933 } 934 935 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more()) 936 return NETDEV_TX_OK; 937 938 gve_tx_put_doorbell_dqo(priv, tx->q_resources, tx->dqo_tx.tail); 939 return NETDEV_TX_OK; 940 } 941 942 static void add_to_list(struct gve_tx_ring *tx, struct gve_index_list *list, 943 struct gve_tx_pending_packet_dqo *pending_packet) 944 { 945 s16 old_tail, index; 946 947 index = pending_packet - tx->dqo.pending_packets; 948 old_tail = list->tail; 949 list->tail = index; 950 if (old_tail == -1) 951 list->head = index; 952 else 953 tx->dqo.pending_packets[old_tail].next = index; 954 955 pending_packet->next = -1; 956 pending_packet->prev = old_tail; 957 } 958 959 static void remove_from_list(struct gve_tx_ring *tx, 960 struct gve_index_list *list, 961 struct gve_tx_pending_packet_dqo *pkt) 962 { 963 s16 prev_index, next_index; 964 965 prev_index = pkt->prev; 966 next_index = pkt->next; 967 968 if (prev_index == -1) { 969 /* Node is head */ 970 list->head = next_index; 971 } else { 972 tx->dqo.pending_packets[prev_index].next = next_index; 973 } 974 if (next_index == -1) { 975 /* Node is tail */ 976 list->tail = prev_index; 977 } else { 978 tx->dqo.pending_packets[next_index].prev = prev_index; 979 } 980 } 981 982 static void gve_unmap_packet(struct device *dev, 983 struct gve_tx_pending_packet_dqo *pkt) 984 { 985 int i; 986 987 /* SKB linear portion is guaranteed to be mapped */ 988 dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]), 989 dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE); 990 for (i = 1; i < pkt->num_bufs; i++) { 991 dma_unmap_page(dev, dma_unmap_addr(pkt, dma[i]), 992 dma_unmap_len(pkt, len[i]), DMA_TO_DEVICE); 993 } 994 pkt->num_bufs = 0; 995 } 996 997 /* Completion types and expected behavior: 998 * No Miss compl + Packet compl = Packet completed normally. 999 * Miss compl + Re-inject compl = Packet completed normally. 1000 * No Miss compl + Re-inject compl = Skipped i.e. packet not completed. 1001 * Miss compl + Packet compl = Skipped i.e. packet not completed. 1002 */ 1003 static void gve_handle_packet_completion(struct gve_priv *priv, 1004 struct gve_tx_ring *tx, bool is_napi, 1005 u16 compl_tag, u64 *bytes, u64 *pkts, 1006 bool is_reinjection) 1007 { 1008 struct gve_tx_pending_packet_dqo *pending_packet; 1009 1010 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { 1011 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", 1012 priv->dev->name, (int)compl_tag); 1013 return; 1014 } 1015 1016 pending_packet = &tx->dqo.pending_packets[compl_tag]; 1017 1018 if (unlikely(is_reinjection)) { 1019 if (unlikely(pending_packet->state == 1020 GVE_PACKET_STATE_TIMED_OUT_COMPL)) { 1021 net_err_ratelimited("%s: Re-injection completion: %d received after timeout.\n", 1022 priv->dev->name, (int)compl_tag); 1023 /* Packet was already completed as a result of timeout, 1024 * so just remove from list and free pending packet. 1025 */ 1026 remove_from_list(tx, 1027 &tx->dqo_compl.timed_out_completions, 1028 pending_packet); 1029 gve_free_pending_packet(tx, pending_packet); 1030 return; 1031 } 1032 if (unlikely(pending_packet->state != 1033 GVE_PACKET_STATE_PENDING_REINJECT_COMPL)) { 1034 /* No outstanding miss completion but packet allocated 1035 * implies packet receives a re-injection completion 1036 * without a prior miss completion. Return without 1037 * completing the packet. 1038 */ 1039 net_err_ratelimited("%s: Re-injection completion received without corresponding miss completion: %d\n", 1040 priv->dev->name, (int)compl_tag); 1041 return; 1042 } 1043 remove_from_list(tx, &tx->dqo_compl.miss_completions, 1044 pending_packet); 1045 } else { 1046 /* Packet is allocated but not a pending data completion. */ 1047 if (unlikely(pending_packet->state != 1048 GVE_PACKET_STATE_PENDING_DATA_COMPL)) { 1049 net_err_ratelimited("%s: No pending data completion: %d\n", 1050 priv->dev->name, (int)compl_tag); 1051 return; 1052 } 1053 } 1054 tx->dqo_tx.completed_packet_desc_cnt += pending_packet->num_bufs; 1055 if (tx->dqo.qpl) 1056 gve_free_tx_qpl_bufs(tx, pending_packet); 1057 else 1058 gve_unmap_packet(tx->dev, pending_packet); 1059 1060 *bytes += pending_packet->skb->len; 1061 (*pkts)++; 1062 napi_consume_skb(pending_packet->skb, is_napi); 1063 pending_packet->skb = NULL; 1064 gve_free_pending_packet(tx, pending_packet); 1065 } 1066 1067 static void gve_handle_miss_completion(struct gve_priv *priv, 1068 struct gve_tx_ring *tx, u16 compl_tag, 1069 u64 *bytes, u64 *pkts) 1070 { 1071 struct gve_tx_pending_packet_dqo *pending_packet; 1072 1073 if (unlikely(compl_tag >= tx->dqo.num_pending_packets)) { 1074 net_err_ratelimited("%s: Invalid TX completion tag: %d\n", 1075 priv->dev->name, (int)compl_tag); 1076 return; 1077 } 1078 1079 pending_packet = &tx->dqo.pending_packets[compl_tag]; 1080 if (unlikely(pending_packet->state != 1081 GVE_PACKET_STATE_PENDING_DATA_COMPL)) { 1082 net_err_ratelimited("%s: Unexpected packet state: %d for completion tag : %d\n", 1083 priv->dev->name, (int)pending_packet->state, 1084 (int)compl_tag); 1085 return; 1086 } 1087 1088 pending_packet->state = GVE_PACKET_STATE_PENDING_REINJECT_COMPL; 1089 /* jiffies can wraparound but time comparisons can handle overflows. */ 1090 pending_packet->timeout_jiffies = 1091 jiffies + 1092 msecs_to_jiffies(GVE_REINJECT_COMPL_TIMEOUT * 1093 MSEC_PER_SEC); 1094 add_to_list(tx, &tx->dqo_compl.miss_completions, pending_packet); 1095 1096 *bytes += pending_packet->skb->len; 1097 (*pkts)++; 1098 } 1099 1100 static void remove_miss_completions(struct gve_priv *priv, 1101 struct gve_tx_ring *tx) 1102 { 1103 struct gve_tx_pending_packet_dqo *pending_packet; 1104 s16 next_index; 1105 1106 next_index = tx->dqo_compl.miss_completions.head; 1107 while (next_index != -1) { 1108 pending_packet = &tx->dqo.pending_packets[next_index]; 1109 next_index = pending_packet->next; 1110 /* Break early because packets should timeout in order. */ 1111 if (time_is_after_jiffies(pending_packet->timeout_jiffies)) 1112 break; 1113 1114 remove_from_list(tx, &tx->dqo_compl.miss_completions, 1115 pending_packet); 1116 /* Unmap/free TX buffers and free skb but do not unallocate packet i.e. 1117 * the completion tag is not freed to ensure that the driver 1118 * can take appropriate action if a corresponding valid 1119 * completion is received later. 1120 */ 1121 if (tx->dqo.qpl) 1122 gve_free_tx_qpl_bufs(tx, pending_packet); 1123 else 1124 gve_unmap_packet(tx->dev, pending_packet); 1125 1126 /* This indicates the packet was dropped. */ 1127 dev_kfree_skb_any(pending_packet->skb); 1128 pending_packet->skb = NULL; 1129 tx->dropped_pkt++; 1130 net_err_ratelimited("%s: No reinjection completion was received for: %d.\n", 1131 priv->dev->name, 1132 (int)(pending_packet - tx->dqo.pending_packets)); 1133 1134 pending_packet->state = GVE_PACKET_STATE_TIMED_OUT_COMPL; 1135 pending_packet->timeout_jiffies = 1136 jiffies + 1137 msecs_to_jiffies(GVE_DEALLOCATE_COMPL_TIMEOUT * 1138 MSEC_PER_SEC); 1139 /* Maintain pending packet in another list so the packet can be 1140 * unallocated at a later time. 1141 */ 1142 add_to_list(tx, &tx->dqo_compl.timed_out_completions, 1143 pending_packet); 1144 } 1145 } 1146 1147 static void remove_timed_out_completions(struct gve_priv *priv, 1148 struct gve_tx_ring *tx) 1149 { 1150 struct gve_tx_pending_packet_dqo *pending_packet; 1151 s16 next_index; 1152 1153 next_index = tx->dqo_compl.timed_out_completions.head; 1154 while (next_index != -1) { 1155 pending_packet = &tx->dqo.pending_packets[next_index]; 1156 next_index = pending_packet->next; 1157 /* Break early because packets should timeout in order. */ 1158 if (time_is_after_jiffies(pending_packet->timeout_jiffies)) 1159 break; 1160 1161 remove_from_list(tx, &tx->dqo_compl.timed_out_completions, 1162 pending_packet); 1163 gve_free_pending_packet(tx, pending_packet); 1164 } 1165 } 1166 1167 int gve_clean_tx_done_dqo(struct gve_priv *priv, struct gve_tx_ring *tx, 1168 struct napi_struct *napi) 1169 { 1170 u64 reinject_compl_bytes = 0; 1171 u64 reinject_compl_pkts = 0; 1172 int num_descs_cleaned = 0; 1173 u64 miss_compl_bytes = 0; 1174 u64 miss_compl_pkts = 0; 1175 u64 pkt_compl_bytes = 0; 1176 u64 pkt_compl_pkts = 0; 1177 1178 /* Limit in order to avoid blocking for too long */ 1179 while (!napi || pkt_compl_pkts < napi->weight) { 1180 struct gve_tx_compl_desc *compl_desc = 1181 &tx->dqo.compl_ring[tx->dqo_compl.head]; 1182 u16 type; 1183 1184 if (compl_desc->generation == tx->dqo_compl.cur_gen_bit) 1185 break; 1186 1187 /* Prefetch the next descriptor. */ 1188 prefetch(&tx->dqo.compl_ring[(tx->dqo_compl.head + 1) & 1189 tx->dqo.complq_mask]); 1190 1191 /* Do not read data until we own the descriptor */ 1192 dma_rmb(); 1193 type = compl_desc->type; 1194 1195 if (type == GVE_COMPL_TYPE_DQO_DESC) { 1196 /* This is the last descriptor fetched by HW plus one */ 1197 u16 tx_head = le16_to_cpu(compl_desc->tx_head); 1198 1199 atomic_set_release(&tx->dqo_compl.hw_tx_head, tx_head); 1200 } else if (type == GVE_COMPL_TYPE_DQO_PKT) { 1201 u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); 1202 if (compl_tag & GVE_ALT_MISS_COMPL_BIT) { 1203 compl_tag &= ~GVE_ALT_MISS_COMPL_BIT; 1204 gve_handle_miss_completion(priv, tx, compl_tag, 1205 &miss_compl_bytes, 1206 &miss_compl_pkts); 1207 } else { 1208 gve_handle_packet_completion(priv, tx, !!napi, 1209 compl_tag, 1210 &pkt_compl_bytes, 1211 &pkt_compl_pkts, 1212 false); 1213 } 1214 } else if (type == GVE_COMPL_TYPE_DQO_MISS) { 1215 u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); 1216 1217 gve_handle_miss_completion(priv, tx, compl_tag, 1218 &miss_compl_bytes, 1219 &miss_compl_pkts); 1220 } else if (type == GVE_COMPL_TYPE_DQO_REINJECTION) { 1221 u16 compl_tag = le16_to_cpu(compl_desc->completion_tag); 1222 1223 gve_handle_packet_completion(priv, tx, !!napi, 1224 compl_tag, 1225 &reinject_compl_bytes, 1226 &reinject_compl_pkts, 1227 true); 1228 } 1229 1230 tx->dqo_compl.head = 1231 (tx->dqo_compl.head + 1) & tx->dqo.complq_mask; 1232 /* Flip the generation bit when we wrap around */ 1233 tx->dqo_compl.cur_gen_bit ^= tx->dqo_compl.head == 0; 1234 num_descs_cleaned++; 1235 } 1236 1237 netdev_tx_completed_queue(tx->netdev_txq, 1238 pkt_compl_pkts + miss_compl_pkts, 1239 pkt_compl_bytes + miss_compl_bytes); 1240 1241 remove_miss_completions(priv, tx); 1242 remove_timed_out_completions(priv, tx); 1243 1244 u64_stats_update_begin(&tx->statss); 1245 tx->bytes_done += pkt_compl_bytes + reinject_compl_bytes; 1246 tx->pkt_done += pkt_compl_pkts + reinject_compl_pkts; 1247 u64_stats_update_end(&tx->statss); 1248 return num_descs_cleaned; 1249 } 1250 1251 bool gve_tx_poll_dqo(struct gve_notify_block *block, bool do_clean) 1252 { 1253 struct gve_tx_compl_desc *compl_desc; 1254 struct gve_tx_ring *tx = block->tx; 1255 struct gve_priv *priv = block->priv; 1256 1257 if (do_clean) { 1258 int num_descs_cleaned = gve_clean_tx_done_dqo(priv, tx, 1259 &block->napi); 1260 1261 /* Sync with queue being stopped in `gve_maybe_stop_tx_dqo()` */ 1262 mb(); 1263 1264 if (netif_tx_queue_stopped(tx->netdev_txq) && 1265 num_descs_cleaned > 0) { 1266 tx->wake_queue++; 1267 netif_tx_wake_queue(tx->netdev_txq); 1268 } 1269 } 1270 1271 /* Return true if we still have work. */ 1272 compl_desc = &tx->dqo.compl_ring[tx->dqo_compl.head]; 1273 return compl_desc->generation != tx->dqo_compl.cur_gen_bit; 1274 } 1275