1 /* 2 * QEMU TX packets abstractions 3 * 4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com) 5 * 6 * Developed by Daynix Computing LTD (http://www.daynix.com) 7 * 8 * Authors: 9 * Dmitry Fleytman <dmitry@daynix.com> 10 * Tamir Shomer <tamirs@daynix.com> 11 * Yan Vugenfirer <yan@daynix.com> 12 * 13 * This work is licensed under the terms of the GNU GPL, version 2 or later. 14 * See the COPYING file in the top-level directory. 15 * 16 */ 17 18 #include "qemu/osdep.h" 19 #include "net_tx_pkt.h" 20 #include "net/eth.h" 21 #include "net/checksum.h" 22 #include "net/tap.h" 23 #include "net/net.h" 24 #include "hw/pci/pci_device.h" 25 26 enum { 27 NET_TX_PKT_VHDR_FRAG = 0, 28 NET_TX_PKT_L2HDR_FRAG, 29 NET_TX_PKT_L3HDR_FRAG, 30 NET_TX_PKT_PL_START_FRAG 31 }; 32 33 /* TX packet private context */ 34 struct NetTxPkt { 35 PCIDevice *pci_dev; 36 37 struct virtio_net_hdr virt_hdr; 38 39 struct iovec *raw; 40 uint32_t raw_frags; 41 uint32_t max_raw_frags; 42 43 struct iovec *vec; 44 45 uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN]; 46 uint8_t l3_hdr[ETH_MAX_IP_DGRAM_LEN]; 47 48 uint32_t payload_len; 49 50 uint32_t payload_frags; 51 uint32_t max_payload_frags; 52 53 uint16_t hdr_len; 54 eth_pkt_types_e packet_type; 55 uint8_t l4proto; 56 }; 57 58 void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev, 59 uint32_t max_frags) 60 { 61 struct NetTxPkt *p = g_malloc0(sizeof *p); 62 63 p->pci_dev = pci_dev; 64 65 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG); 66 67 p->raw = g_new(struct iovec, max_frags); 68 69 p->max_payload_frags = max_frags; 70 p->max_raw_frags = max_frags; 71 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr; 72 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr; 73 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr; 74 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr; 75 76 *pkt = p; 77 } 78 79 void net_tx_pkt_uninit(struct NetTxPkt *pkt) 80 { 81 if (pkt) { 82 g_free(pkt->vec); 83 g_free(pkt->raw); 84 g_free(pkt); 85 } 86 } 87 88 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt) 89 { 90 uint16_t csum; 91 assert(pkt); 92 struct ip_header *ip_hdr; 93 ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; 94 95 ip_hdr->ip_len = cpu_to_be16(pkt->payload_len + 96 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 97 98 ip_hdr->ip_sum = 0; 99 csum = net_raw_checksum((uint8_t *)ip_hdr, 100 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 101 ip_hdr->ip_sum = cpu_to_be16(csum); 102 } 103 104 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt) 105 { 106 uint16_t csum; 107 uint32_t cntr, cso; 108 assert(pkt); 109 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 110 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; 111 112 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len > 113 ETH_MAX_IP_DGRAM_LEN) { 114 return; 115 } 116 117 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 118 gso_type == VIRTIO_NET_HDR_GSO_UDP) { 119 /* Calculate IP header checksum */ 120 net_tx_pkt_update_ip_hdr_checksum(pkt); 121 122 /* Calculate IP pseudo header checksum */ 123 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso); 124 csum = cpu_to_be16(~net_checksum_finish(cntr)); 125 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 126 /* Calculate IP pseudo header checksum */ 127 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len, 128 IP_PROTO_TCP, &cso); 129 csum = cpu_to_be16(~net_checksum_finish(cntr)); 130 } else { 131 return; 132 } 133 134 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags, 135 pkt->virt_hdr.csum_offset, &csum, sizeof(csum)); 136 } 137 138 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt) 139 { 140 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len + 141 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len; 142 } 143 144 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt) 145 { 146 struct iovec *l2_hdr, *l3_hdr; 147 size_t bytes_read; 148 size_t full_ip6hdr_len; 149 uint16_t l3_proto; 150 151 assert(pkt); 152 153 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 154 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 155 156 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base, 157 ETH_MAX_L2_HDR_LEN); 158 if (bytes_read < sizeof(struct eth_header)) { 159 l2_hdr->iov_len = 0; 160 return false; 161 } 162 163 l2_hdr->iov_len = sizeof(struct eth_header); 164 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) { 165 case ETH_P_VLAN: 166 l2_hdr->iov_len += sizeof(struct vlan_header); 167 break; 168 case ETH_P_DVLAN: 169 l2_hdr->iov_len += 2 * sizeof(struct vlan_header); 170 break; 171 } 172 173 if (bytes_read < l2_hdr->iov_len) { 174 l2_hdr->iov_len = 0; 175 l3_hdr->iov_len = 0; 176 pkt->packet_type = ETH_PKT_UCAST; 177 return false; 178 } else { 179 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN; 180 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base); 181 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base); 182 } 183 184 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len); 185 186 switch (l3_proto) { 187 case ETH_P_IP: 188 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 189 l3_hdr->iov_base, sizeof(struct ip_header)); 190 191 if (bytes_read < sizeof(struct ip_header)) { 192 l3_hdr->iov_len = 0; 193 return false; 194 } 195 196 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base); 197 198 if (l3_hdr->iov_len < sizeof(struct ip_header)) { 199 l3_hdr->iov_len = 0; 200 return false; 201 } 202 203 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base); 204 205 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) { 206 /* copy optional IPv4 header data if any*/ 207 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 208 l2_hdr->iov_len + sizeof(struct ip_header), 209 l3_hdr->iov_base + sizeof(struct ip_header), 210 l3_hdr->iov_len - sizeof(struct ip_header)); 211 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) { 212 l3_hdr->iov_len = 0; 213 return false; 214 } 215 } 216 217 break; 218 219 case ETH_P_IPV6: 220 { 221 eth_ip6_hdr_info hdrinfo; 222 223 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 224 &hdrinfo)) { 225 l3_hdr->iov_len = 0; 226 return false; 227 } 228 229 pkt->l4proto = hdrinfo.l4proto; 230 full_ip6hdr_len = hdrinfo.full_hdr_len; 231 232 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) { 233 l3_hdr->iov_len = 0; 234 return false; 235 } 236 237 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 238 l3_hdr->iov_base, full_ip6hdr_len); 239 240 if (bytes_read < full_ip6hdr_len) { 241 l3_hdr->iov_len = 0; 242 return false; 243 } else { 244 l3_hdr->iov_len = full_ip6hdr_len; 245 } 246 break; 247 } 248 default: 249 l3_hdr->iov_len = 0; 250 break; 251 } 252 253 net_tx_pkt_calculate_hdr_len(pkt); 254 return true; 255 } 256 257 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt) 258 { 259 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len; 260 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 261 pkt->max_payload_frags, 262 pkt->raw, pkt->raw_frags, 263 pkt->hdr_len, pkt->payload_len); 264 } 265 266 bool net_tx_pkt_parse(struct NetTxPkt *pkt) 267 { 268 if (net_tx_pkt_parse_headers(pkt)) { 269 net_tx_pkt_rebuild_payload(pkt); 270 return true; 271 } else { 272 return false; 273 } 274 } 275 276 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt) 277 { 278 assert(pkt); 279 return &pkt->virt_hdr; 280 } 281 282 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt, 283 bool tso_enable) 284 { 285 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE; 286 uint16_t l3_proto; 287 288 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1, 289 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len); 290 291 if (!tso_enable) { 292 goto func_exit; 293 } 294 295 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 296 pkt->l4proto); 297 298 func_exit: 299 return rc; 300 } 301 302 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, 303 bool csum_enable, uint32_t gso_size) 304 { 305 struct tcp_hdr l4hdr; 306 size_t bytes_read; 307 assert(pkt); 308 309 /* csum has to be enabled if tso is. */ 310 assert(csum_enable || !tso_enable); 311 312 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable); 313 314 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 315 case VIRTIO_NET_HDR_GSO_NONE: 316 pkt->virt_hdr.hdr_len = 0; 317 pkt->virt_hdr.gso_size = 0; 318 break; 319 320 case VIRTIO_NET_HDR_GSO_UDP: 321 pkt->virt_hdr.gso_size = gso_size; 322 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header); 323 break; 324 325 case VIRTIO_NET_HDR_GSO_TCPV4: 326 case VIRTIO_NET_HDR_GSO_TCPV6: 327 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 328 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr)); 329 if (bytes_read < sizeof(l4hdr) || 330 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) { 331 return false; 332 } 333 334 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t); 335 pkt->virt_hdr.gso_size = gso_size; 336 break; 337 338 default: 339 g_assert_not_reached(); 340 } 341 342 if (csum_enable) { 343 switch (pkt->l4proto) { 344 case IP_PROTO_TCP: 345 if (pkt->payload_len < sizeof(struct tcp_hdr)) { 346 return false; 347 } 348 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 349 pkt->virt_hdr.csum_start = pkt->hdr_len; 350 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum); 351 break; 352 case IP_PROTO_UDP: 353 if (pkt->payload_len < sizeof(struct udp_hdr)) { 354 return false; 355 } 356 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 357 pkt->virt_hdr.csum_start = pkt->hdr_len; 358 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum); 359 break; 360 default: 361 break; 362 } 363 } 364 365 return true; 366 } 367 368 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, 369 uint16_t vlan, uint16_t vlan_ethtype) 370 { 371 bool is_new; 372 assert(pkt); 373 374 eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, 375 vlan, vlan_ethtype, &is_new); 376 377 /* update l2hdrlen */ 378 if (is_new) { 379 pkt->hdr_len += sizeof(struct vlan_header); 380 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len += 381 sizeof(struct vlan_header); 382 } 383 } 384 385 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa, 386 size_t len) 387 { 388 hwaddr mapped_len = 0; 389 struct iovec *ventry; 390 assert(pkt); 391 392 if (pkt->raw_frags >= pkt->max_raw_frags) { 393 return false; 394 } 395 396 if (!len) { 397 return true; 398 } 399 400 ventry = &pkt->raw[pkt->raw_frags]; 401 mapped_len = len; 402 403 ventry->iov_base = pci_dma_map(pkt->pci_dev, pa, 404 &mapped_len, DMA_DIRECTION_TO_DEVICE); 405 406 if ((ventry->iov_base != NULL) && (len == mapped_len)) { 407 ventry->iov_len = mapped_len; 408 pkt->raw_frags++; 409 return true; 410 } else { 411 return false; 412 } 413 } 414 415 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt) 416 { 417 return pkt->raw_frags > 0; 418 } 419 420 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt) 421 { 422 assert(pkt); 423 424 return pkt->packet_type; 425 } 426 427 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt) 428 { 429 assert(pkt); 430 431 return pkt->hdr_len + pkt->payload_len; 432 } 433 434 void net_tx_pkt_dump(struct NetTxPkt *pkt) 435 { 436 #ifdef NET_TX_PKT_DEBUG 437 assert(pkt); 438 439 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, " 440 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type, 441 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, 442 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len); 443 #endif 444 } 445 446 void net_tx_pkt_reset(struct NetTxPkt *pkt, PCIDevice *pci_dev) 447 { 448 int i; 449 450 /* no assert, as reset can be called before tx_pkt_init */ 451 if (!pkt) { 452 return; 453 } 454 455 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr)); 456 457 assert(pkt->vec); 458 459 pkt->payload_len = 0; 460 pkt->payload_frags = 0; 461 462 if (pkt->max_raw_frags > 0) { 463 assert(pkt->raw); 464 for (i = 0; i < pkt->raw_frags; i++) { 465 assert(pkt->raw[i].iov_base); 466 pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base, 467 pkt->raw[i].iov_len, DMA_DIRECTION_TO_DEVICE, 0); 468 } 469 } 470 pkt->pci_dev = pci_dev; 471 pkt->raw_frags = 0; 472 473 pkt->hdr_len = 0; 474 pkt->l4proto = 0; 475 } 476 477 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, 478 struct iovec *iov, uint32_t iov_len, 479 uint16_t csl) 480 { 481 uint32_t csum_cntr; 482 uint16_t csum = 0; 483 uint32_t cso; 484 /* num of iovec without vhdr */ 485 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset; 486 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len); 487 488 /* Put zero to checksum field */ 489 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 490 491 /* Calculate L4 TCP/UDP checksum */ 492 csum_cntr = 0; 493 cso = 0; 494 /* add pseudo header to csum */ 495 if (l3_proto == ETH_P_IP) { 496 csum_cntr = eth_calc_ip4_pseudo_hdr_csum( 497 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 498 csl, &cso); 499 } else if (l3_proto == ETH_P_IPV6) { 500 csum_cntr = eth_calc_ip6_pseudo_hdr_csum( 501 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 502 csl, pkt->l4proto, &cso); 503 } 504 505 /* data checksum */ 506 csum_cntr += 507 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso); 508 509 /* Put the checksum obtained into the packet */ 510 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr)); 511 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 512 } 513 514 #define NET_MAX_FRAG_SG_LIST (64) 515 516 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, 517 int *src_idx, size_t *src_offset, size_t src_len, 518 struct iovec *dst, int *dst_idx) 519 { 520 size_t fetched = 0; 521 struct iovec *src = pkt->vec; 522 523 while (fetched < src_len) { 524 525 /* no more place in fragment iov */ 526 if (*dst_idx == NET_MAX_FRAG_SG_LIST) { 527 break; 528 } 529 530 /* no more data in iovec */ 531 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) { 532 break; 533 } 534 535 536 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset; 537 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset, 538 src_len - fetched); 539 540 *src_offset += dst[*dst_idx].iov_len; 541 fetched += dst[*dst_idx].iov_len; 542 543 if (*src_offset == src[*src_idx].iov_len) { 544 *src_offset = 0; 545 (*src_idx)++; 546 } 547 548 (*dst_idx)++; 549 } 550 551 return fetched; 552 } 553 554 static void net_tx_pkt_sendv( 555 void *opaque, const struct iovec *iov, int iov_cnt, 556 const struct iovec *virt_iov, int virt_iov_cnt) 557 { 558 NetClientState *nc = opaque; 559 560 if (qemu_get_using_vnet_hdr(nc->peer)) { 561 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt); 562 } else { 563 qemu_sendv_packet(nc, iov, iov_cnt); 564 } 565 } 566 567 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt, 568 struct iovec *fragment, 569 int *pl_idx, 570 size_t *l4hdr_len, 571 int *src_idx, 572 size_t *src_offset, 573 size_t *src_len) 574 { 575 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG; 576 size_t bytes_read = 0; 577 struct tcp_hdr *th; 578 579 if (!pkt->payload_frags) { 580 return false; 581 } 582 583 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len; 584 l4->iov_base = g_malloc(l4->iov_len); 585 586 *src_idx = NET_TX_PKT_PL_START_FRAG; 587 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) { 588 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 589 pkt->vec[*src_idx].iov_len); 590 591 bytes_read += pkt->vec[*src_idx].iov_len; 592 593 (*src_idx)++; 594 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) { 595 g_free(l4->iov_base); 596 return false; 597 } 598 } 599 600 *src_offset = l4->iov_len - bytes_read; 601 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 602 *src_offset); 603 604 th = l4->iov_base; 605 th->th_flags &= ~(TH_FIN | TH_PUSH); 606 607 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1; 608 *l4hdr_len = l4->iov_len; 609 *src_len = pkt->virt_hdr.gso_size; 610 611 return true; 612 } 613 614 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment) 615 { 616 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base); 617 } 618 619 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt, 620 struct iovec *fragment, 621 size_t fragment_len, 622 uint8_t gso_type) 623 { 624 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 625 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 626 struct ip_header *ip = l3hdr->iov_base; 627 struct ip6_header *ip6 = l3hdr->iov_base; 628 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len; 629 630 switch (gso_type) { 631 case VIRTIO_NET_HDR_GSO_TCPV4: 632 ip->ip_len = cpu_to_be16(len); 633 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 634 break; 635 636 case VIRTIO_NET_HDR_GSO_TCPV6: 637 len -= sizeof(struct ip6_header); 638 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len); 639 break; 640 } 641 } 642 643 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt, 644 struct iovec *fragment, 645 size_t fragment_len, 646 uint8_t gso_type) 647 { 648 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 649 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 650 struct ip_header *ip = l3hdr->iov_base; 651 struct tcp_hdr *th = l4hdr->iov_base; 652 653 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) { 654 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1); 655 } 656 657 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len); 658 th->th_flags &= ~TH_CWR; 659 } 660 661 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt, 662 int *pl_idx, 663 size_t *l4hdr_len, 664 int *src_idx, size_t *src_offset, 665 size_t *src_len) 666 { 667 *pl_idx = NET_TX_PKT_PL_START_FRAG; 668 *l4hdr_len = 0; 669 *src_idx = NET_TX_PKT_PL_START_FRAG; 670 *src_offset = 0; 671 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size); 672 } 673 674 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, 675 struct iovec *fragment, 676 size_t fragment_offset, 677 size_t fragment_len) 678 { 679 bool more_frags = fragment_offset + fragment_len < pkt->payload_len; 680 uint16_t orig_flags; 681 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 682 struct ip_header *ip = l3hdr->iov_base; 683 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE; 684 uint16_t new_ip_off; 685 686 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0); 687 assert((frag_off_units & ~IP_OFFMASK) == 0); 688 689 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF); 690 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); 691 ip->ip_off = cpu_to_be16(new_ip_off); 692 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len); 693 694 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 695 } 696 697 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, 698 NetTxPktCallback callback, 699 void *context) 700 { 701 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 702 703 struct iovec fragment[NET_MAX_FRAG_SG_LIST]; 704 size_t fragment_len; 705 size_t l4hdr_len; 706 size_t src_len; 707 708 int src_idx, dst_idx, pl_idx; 709 size_t src_offset; 710 size_t fragment_offset = 0; 711 struct virtio_net_hdr virt_hdr = { 712 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ? 713 VIRTIO_NET_HDR_F_DATA_VALID : 0 714 }; 715 716 /* Copy headers */ 717 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr; 718 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr); 719 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 720 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 721 722 switch (gso_type) { 723 case VIRTIO_NET_HDR_GSO_TCPV4: 724 case VIRTIO_NET_HDR_GSO_TCPV6: 725 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len, 726 &src_idx, &src_offset, &src_len)) { 727 return false; 728 } 729 break; 730 731 case VIRTIO_NET_HDR_GSO_UDP: 732 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 733 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 734 pkt->payload_len); 735 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len, 736 &src_idx, &src_offset, &src_len); 737 break; 738 739 default: 740 abort(); 741 } 742 743 /* Put as much data as possible and send */ 744 while (true) { 745 dst_idx = pl_idx; 746 fragment_len = net_tx_pkt_fetch_fragment(pkt, 747 &src_idx, &src_offset, src_len, fragment, &dst_idx); 748 if (!fragment_len) { 749 break; 750 } 751 752 switch (gso_type) { 753 case VIRTIO_NET_HDR_GSO_TCPV4: 754 case VIRTIO_NET_HDR_GSO_TCPV6: 755 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type); 756 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG, 757 dst_idx - NET_TX_PKT_L2HDR_FRAG, 758 l4hdr_len + fragment_len); 759 break; 760 761 case VIRTIO_NET_HDR_GSO_UDP: 762 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset, 763 fragment_len); 764 break; 765 } 766 767 callback(context, 768 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG, 769 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG); 770 771 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 772 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 773 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len, 774 gso_type); 775 } 776 777 fragment_offset += fragment_len; 778 } 779 780 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 781 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 782 net_tx_pkt_tcp_fragment_deinit(fragment); 783 } 784 785 return true; 786 } 787 788 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) 789 { 790 bool offload = qemu_get_using_vnet_hdr(nc->peer); 791 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc); 792 } 793 794 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, 795 NetTxPktCallback callback, void *context) 796 { 797 assert(pkt); 798 799 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 800 801 /* 802 * Since underlying infrastructure does not support IP datagrams longer 803 * than 64K we should drop such packets and don't even try to send 804 */ 805 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) { 806 if (pkt->payload_len > 807 ETH_MAX_IP_DGRAM_LEN - 808 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) { 809 return false; 810 } 811 } 812 813 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) { 814 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 815 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 816 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 817 pkt->payload_len); 818 } 819 820 net_tx_pkt_fix_ip6_payload_len(pkt); 821 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG, 822 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG, 823 pkt->vec + NET_TX_PKT_VHDR_FRAG, 824 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG); 825 return true; 826 } 827 828 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context); 829 } 830 831 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt) 832 { 833 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 834 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) { 835 struct ip6_header *ip6 = (struct ip6_header *) pkt->l3_hdr; 836 /* 837 * TODO: if qemu would support >64K packets - add jumbo option check 838 * something like that: 839 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {' 840 */ 841 if (ip6->ip6_plen == 0) { 842 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) { 843 ip6->ip6_plen = htons(pkt->payload_len); 844 } 845 /* 846 * TODO: if qemu would support >64K packets 847 * add jumbo option for packets greater then 65,535 bytes 848 */ 849 } 850 } 851 } 852