1 /* 2 * QEMU TX packets abstractions 3 * 4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com) 5 * 6 * Developed by Daynix Computing LTD (http://www.daynix.com) 7 * 8 * Authors: 9 * Dmitry Fleytman <dmitry@daynix.com> 10 * Tamir Shomer <tamirs@daynix.com> 11 * Yan Vugenfirer <yan@daynix.com> 12 * 13 * This work is licensed under the terms of the GNU GPL, version 2 or later. 14 * See the COPYING file in the top-level directory. 15 * 16 */ 17 18 #include "qemu/osdep.h" 19 #include "qemu/crc32c.h" 20 #include "net/eth.h" 21 #include "net/checksum.h" 22 #include "net/tap.h" 23 #include "net/net.h" 24 #include "hw/pci/pci_device.h" 25 #include "net_tx_pkt.h" 26 27 enum { 28 NET_TX_PKT_VHDR_FRAG = 0, 29 NET_TX_PKT_L2HDR_FRAG, 30 NET_TX_PKT_L3HDR_FRAG, 31 NET_TX_PKT_PL_START_FRAG 32 }; 33 34 /* TX packet private context */ 35 struct NetTxPkt { 36 struct virtio_net_hdr virt_hdr; 37 38 struct iovec *raw; 39 uint32_t raw_frags; 40 uint32_t max_raw_frags; 41 42 struct iovec *vec; 43 44 struct { 45 struct eth_header eth; 46 struct vlan_header vlan[3]; 47 } l2_hdr; 48 union { 49 struct ip_header ip; 50 struct ip6_header ip6; 51 uint8_t octets[ETH_MAX_IP_DGRAM_LEN]; 52 } l3_hdr; 53 54 uint32_t payload_len; 55 56 uint32_t payload_frags; 57 uint32_t max_payload_frags; 58 59 uint16_t hdr_len; 60 eth_pkt_types_e packet_type; 61 uint8_t l4proto; 62 }; 63 64 void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags) 65 { 66 struct NetTxPkt *p = g_malloc0(sizeof *p); 67 68 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG); 69 70 p->raw = g_new(struct iovec, max_frags); 71 72 p->max_payload_frags = max_frags; 73 p->max_raw_frags = max_frags; 74 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr; 75 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr; 76 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr; 77 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr; 78 79 *pkt = p; 80 } 81 82 void net_tx_pkt_uninit(struct NetTxPkt *pkt) 83 { 84 if (pkt) { 85 g_free(pkt->vec); 86 g_free(pkt->raw); 87 g_free(pkt); 88 } 89 } 90 91 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt) 92 { 93 uint16_t csum; 94 assert(pkt); 95 96 pkt->l3_hdr.ip.ip_len = cpu_to_be16(pkt->payload_len + 97 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 98 99 pkt->l3_hdr.ip.ip_sum = 0; 100 csum = net_raw_checksum(pkt->l3_hdr.octets, 101 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 102 pkt->l3_hdr.ip.ip_sum = cpu_to_be16(csum); 103 } 104 105 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt) 106 { 107 uint16_t csum; 108 uint32_t cntr, cso; 109 assert(pkt); 110 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 111 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; 112 113 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len > 114 ETH_MAX_IP_DGRAM_LEN) { 115 return; 116 } 117 118 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 119 gso_type == VIRTIO_NET_HDR_GSO_UDP) { 120 /* Calculate IP header checksum */ 121 net_tx_pkt_update_ip_hdr_checksum(pkt); 122 123 /* Calculate IP pseudo header checksum */ 124 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso); 125 csum = cpu_to_be16(~net_checksum_finish(cntr)); 126 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 127 /* Calculate IP pseudo header checksum */ 128 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len, 129 IP_PROTO_TCP, &cso); 130 csum = cpu_to_be16(~net_checksum_finish(cntr)); 131 } else { 132 return; 133 } 134 135 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags, 136 pkt->virt_hdr.csum_offset, &csum, sizeof(csum)); 137 } 138 139 bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt) 140 { 141 uint32_t csum = 0; 142 struct iovec *pl_start_frag = pkt->vec + NET_TX_PKT_PL_START_FRAG; 143 144 if (iov_size(pl_start_frag, pkt->payload_frags) < 8 + sizeof(csum)) { 145 return false; 146 } 147 148 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) { 149 return false; 150 } 151 152 csum = cpu_to_le32(iov_crc32c(0xffffffff, pl_start_frag, pkt->payload_frags)); 153 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) { 154 return false; 155 } 156 157 return true; 158 } 159 160 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt) 161 { 162 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len + 163 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len; 164 } 165 166 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt) 167 { 168 struct iovec *l2_hdr, *l3_hdr; 169 size_t bytes_read; 170 size_t full_ip6hdr_len; 171 uint16_t l3_proto; 172 173 assert(pkt); 174 175 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 176 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 177 178 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base, 179 ETH_MAX_L2_HDR_LEN); 180 if (bytes_read < sizeof(struct eth_header)) { 181 l2_hdr->iov_len = 0; 182 return false; 183 } 184 185 l2_hdr->iov_len = sizeof(struct eth_header); 186 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) { 187 case ETH_P_VLAN: 188 l2_hdr->iov_len += sizeof(struct vlan_header); 189 break; 190 case ETH_P_DVLAN: 191 l2_hdr->iov_len += 2 * sizeof(struct vlan_header); 192 break; 193 } 194 195 if (bytes_read < l2_hdr->iov_len) { 196 l2_hdr->iov_len = 0; 197 l3_hdr->iov_len = 0; 198 pkt->packet_type = ETH_PKT_UCAST; 199 return false; 200 } else { 201 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN; 202 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base); 203 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base); 204 } 205 206 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len); 207 208 switch (l3_proto) { 209 case ETH_P_IP: 210 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 211 l3_hdr->iov_base, sizeof(struct ip_header)); 212 213 if (bytes_read < sizeof(struct ip_header)) { 214 l3_hdr->iov_len = 0; 215 return false; 216 } 217 218 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base); 219 220 if (l3_hdr->iov_len < sizeof(struct ip_header)) { 221 l3_hdr->iov_len = 0; 222 return false; 223 } 224 225 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base); 226 227 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) { 228 /* copy optional IPv4 header data if any*/ 229 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 230 l2_hdr->iov_len + sizeof(struct ip_header), 231 l3_hdr->iov_base + sizeof(struct ip_header), 232 l3_hdr->iov_len - sizeof(struct ip_header)); 233 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) { 234 l3_hdr->iov_len = 0; 235 return false; 236 } 237 } 238 239 break; 240 241 case ETH_P_IPV6: 242 { 243 eth_ip6_hdr_info hdrinfo; 244 245 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 246 &hdrinfo)) { 247 l3_hdr->iov_len = 0; 248 return false; 249 } 250 251 pkt->l4proto = hdrinfo.l4proto; 252 full_ip6hdr_len = hdrinfo.full_hdr_len; 253 254 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) { 255 l3_hdr->iov_len = 0; 256 return false; 257 } 258 259 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 260 l3_hdr->iov_base, full_ip6hdr_len); 261 262 if (bytes_read < full_ip6hdr_len) { 263 l3_hdr->iov_len = 0; 264 return false; 265 } else { 266 l3_hdr->iov_len = full_ip6hdr_len; 267 } 268 break; 269 } 270 default: 271 l3_hdr->iov_len = 0; 272 break; 273 } 274 275 net_tx_pkt_calculate_hdr_len(pkt); 276 return true; 277 } 278 279 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt) 280 { 281 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len; 282 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 283 pkt->max_payload_frags, 284 pkt->raw, pkt->raw_frags, 285 pkt->hdr_len, pkt->payload_len); 286 } 287 288 bool net_tx_pkt_parse(struct NetTxPkt *pkt) 289 { 290 if (net_tx_pkt_parse_headers(pkt)) { 291 net_tx_pkt_rebuild_payload(pkt); 292 return true; 293 } else { 294 return false; 295 } 296 } 297 298 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt) 299 { 300 assert(pkt); 301 return &pkt->virt_hdr; 302 } 303 304 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt, 305 bool tso_enable) 306 { 307 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE; 308 uint16_t l3_proto; 309 310 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1, 311 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len); 312 313 if (!tso_enable) { 314 goto func_exit; 315 } 316 317 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 318 pkt->l4proto); 319 320 func_exit: 321 return rc; 322 } 323 324 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, 325 bool csum_enable, uint32_t gso_size) 326 { 327 struct tcp_hdr l4hdr; 328 size_t bytes_read; 329 assert(pkt); 330 331 /* csum has to be enabled if tso is. */ 332 assert(csum_enable || !tso_enable); 333 334 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable); 335 336 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 337 case VIRTIO_NET_HDR_GSO_NONE: 338 pkt->virt_hdr.hdr_len = 0; 339 pkt->virt_hdr.gso_size = 0; 340 break; 341 342 case VIRTIO_NET_HDR_GSO_UDP: 343 pkt->virt_hdr.gso_size = gso_size; 344 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header); 345 break; 346 347 case VIRTIO_NET_HDR_GSO_TCPV4: 348 case VIRTIO_NET_HDR_GSO_TCPV6: 349 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 350 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr)); 351 if (bytes_read < sizeof(l4hdr) || 352 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) { 353 return false; 354 } 355 356 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t); 357 pkt->virt_hdr.gso_size = gso_size; 358 break; 359 360 default: 361 g_assert_not_reached(); 362 } 363 364 if (csum_enable) { 365 switch (pkt->l4proto) { 366 case IP_PROTO_TCP: 367 if (pkt->payload_len < sizeof(struct tcp_hdr)) { 368 return false; 369 } 370 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 371 pkt->virt_hdr.csum_start = pkt->hdr_len; 372 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum); 373 break; 374 case IP_PROTO_UDP: 375 if (pkt->payload_len < sizeof(struct udp_hdr)) { 376 return false; 377 } 378 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 379 pkt->virt_hdr.csum_start = pkt->hdr_len; 380 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum); 381 break; 382 default: 383 break; 384 } 385 } 386 387 return true; 388 } 389 390 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, 391 uint16_t vlan, uint16_t vlan_ethtype) 392 { 393 assert(pkt); 394 395 eth_setup_vlan_headers(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, 396 &pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, 397 vlan, vlan_ethtype); 398 399 pkt->hdr_len += sizeof(struct vlan_header); 400 } 401 402 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len) 403 { 404 struct iovec *ventry; 405 assert(pkt); 406 407 if (pkt->raw_frags >= pkt->max_raw_frags) { 408 return false; 409 } 410 411 ventry = &pkt->raw[pkt->raw_frags]; 412 ventry->iov_base = base; 413 ventry->iov_len = len; 414 pkt->raw_frags++; 415 416 return true; 417 } 418 419 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt) 420 { 421 return pkt->raw_frags > 0; 422 } 423 424 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt) 425 { 426 assert(pkt); 427 428 return pkt->packet_type; 429 } 430 431 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt) 432 { 433 assert(pkt); 434 435 return pkt->hdr_len + pkt->payload_len; 436 } 437 438 void net_tx_pkt_dump(struct NetTxPkt *pkt) 439 { 440 #ifdef NET_TX_PKT_DEBUG 441 assert(pkt); 442 443 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, " 444 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type, 445 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, 446 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len); 447 #endif 448 } 449 450 void net_tx_pkt_reset(struct NetTxPkt *pkt, 451 NetTxPktFreeFrag callback, void *context) 452 { 453 int i; 454 455 /* no assert, as reset can be called before tx_pkt_init */ 456 if (!pkt) { 457 return; 458 } 459 460 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr)); 461 462 assert(pkt->vec); 463 464 pkt->payload_len = 0; 465 pkt->payload_frags = 0; 466 467 if (pkt->max_raw_frags > 0) { 468 assert(pkt->raw); 469 for (i = 0; i < pkt->raw_frags; i++) { 470 assert(pkt->raw[i].iov_base); 471 callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len); 472 } 473 } 474 pkt->raw_frags = 0; 475 476 pkt->hdr_len = 0; 477 pkt->l4proto = 0; 478 } 479 480 void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len) 481 { 482 pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0); 483 } 484 485 bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev, 486 dma_addr_t pa, size_t len) 487 { 488 dma_addr_t mapped_len = len; 489 void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE); 490 if (!base) { 491 return false; 492 } 493 494 if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) { 495 net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len); 496 return false; 497 } 498 499 return true; 500 } 501 502 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, 503 struct iovec *iov, uint32_t iov_len, 504 uint16_t csl) 505 { 506 uint32_t csum_cntr; 507 uint16_t csum = 0; 508 uint32_t cso; 509 /* num of iovec without vhdr */ 510 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset; 511 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len); 512 513 /* Put zero to checksum field */ 514 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 515 516 /* Calculate L4 TCP/UDP checksum */ 517 csum_cntr = 0; 518 cso = 0; 519 /* add pseudo header to csum */ 520 if (l3_proto == ETH_P_IP) { 521 csum_cntr = eth_calc_ip4_pseudo_hdr_csum( 522 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 523 csl, &cso); 524 } else if (l3_proto == ETH_P_IPV6) { 525 csum_cntr = eth_calc_ip6_pseudo_hdr_csum( 526 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 527 csl, pkt->l4proto, &cso); 528 } 529 530 /* data checksum */ 531 csum_cntr += 532 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso); 533 534 /* Put the checksum obtained into the packet */ 535 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr)); 536 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 537 } 538 539 #define NET_MAX_FRAG_SG_LIST (64) 540 541 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, 542 int *src_idx, size_t *src_offset, size_t src_len, 543 struct iovec *dst, int *dst_idx) 544 { 545 size_t fetched = 0; 546 struct iovec *src = pkt->vec; 547 548 while (fetched < src_len) { 549 550 /* no more place in fragment iov */ 551 if (*dst_idx == NET_MAX_FRAG_SG_LIST) { 552 break; 553 } 554 555 /* no more data in iovec */ 556 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) { 557 break; 558 } 559 560 561 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset; 562 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset, 563 src_len - fetched); 564 565 *src_offset += dst[*dst_idx].iov_len; 566 fetched += dst[*dst_idx].iov_len; 567 568 if (*src_offset == src[*src_idx].iov_len) { 569 *src_offset = 0; 570 (*src_idx)++; 571 } 572 573 (*dst_idx)++; 574 } 575 576 return fetched; 577 } 578 579 static void net_tx_pkt_sendv( 580 void *opaque, const struct iovec *iov, int iov_cnt, 581 const struct iovec *virt_iov, int virt_iov_cnt) 582 { 583 NetClientState *nc = opaque; 584 585 if (qemu_get_vnet_hdr_len(nc->peer)) { 586 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt); 587 } else { 588 qemu_sendv_packet(nc, iov, iov_cnt); 589 } 590 } 591 592 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt, 593 struct iovec *fragment, 594 int *pl_idx, 595 size_t *l4hdr_len, 596 int *src_idx, 597 size_t *src_offset, 598 size_t *src_len) 599 { 600 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG; 601 size_t bytes_read = 0; 602 struct tcp_hdr *th; 603 604 if (!pkt->payload_frags) { 605 return false; 606 } 607 608 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len; 609 l4->iov_base = g_malloc(l4->iov_len); 610 611 *src_idx = NET_TX_PKT_PL_START_FRAG; 612 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) { 613 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 614 pkt->vec[*src_idx].iov_len); 615 616 bytes_read += pkt->vec[*src_idx].iov_len; 617 618 (*src_idx)++; 619 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) { 620 g_free(l4->iov_base); 621 return false; 622 } 623 } 624 625 *src_offset = l4->iov_len - bytes_read; 626 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 627 *src_offset); 628 629 th = l4->iov_base; 630 th->th_flags &= ~(TH_FIN | TH_PUSH); 631 632 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1; 633 *l4hdr_len = l4->iov_len; 634 *src_len = pkt->virt_hdr.gso_size; 635 636 return true; 637 } 638 639 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment) 640 { 641 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base); 642 } 643 644 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt, 645 struct iovec *fragment, 646 size_t fragment_len, 647 uint8_t gso_type) 648 { 649 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 650 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 651 struct ip_header *ip = l3hdr->iov_base; 652 struct ip6_header *ip6 = l3hdr->iov_base; 653 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len; 654 655 switch (gso_type) { 656 case VIRTIO_NET_HDR_GSO_TCPV4: 657 ip->ip_len = cpu_to_be16(len); 658 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 659 break; 660 661 case VIRTIO_NET_HDR_GSO_TCPV6: 662 len -= sizeof(struct ip6_header); 663 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len); 664 break; 665 } 666 } 667 668 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt, 669 struct iovec *fragment, 670 size_t fragment_len, 671 uint8_t gso_type) 672 { 673 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 674 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 675 struct ip_header *ip = l3hdr->iov_base; 676 struct tcp_hdr *th = l4hdr->iov_base; 677 678 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) { 679 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1); 680 } 681 682 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len); 683 th->th_flags &= ~TH_CWR; 684 } 685 686 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt, 687 int *pl_idx, 688 size_t *l4hdr_len, 689 int *src_idx, size_t *src_offset, 690 size_t *src_len) 691 { 692 *pl_idx = NET_TX_PKT_PL_START_FRAG; 693 *l4hdr_len = 0; 694 *src_idx = NET_TX_PKT_PL_START_FRAG; 695 *src_offset = 0; 696 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size); 697 } 698 699 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, 700 struct iovec *fragment, 701 size_t fragment_offset, 702 size_t fragment_len) 703 { 704 bool more_frags = fragment_offset + fragment_len < pkt->payload_len; 705 uint16_t orig_flags; 706 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 707 struct ip_header *ip = l3hdr->iov_base; 708 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE; 709 uint16_t new_ip_off; 710 711 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0); 712 assert((frag_off_units & ~IP_OFFMASK) == 0); 713 714 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF); 715 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); 716 ip->ip_off = cpu_to_be16(new_ip_off); 717 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len); 718 719 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 720 } 721 722 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, 723 NetTxPktSend callback, 724 void *context) 725 { 726 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 727 728 struct iovec fragment[NET_MAX_FRAG_SG_LIST]; 729 size_t fragment_len; 730 size_t l4hdr_len; 731 size_t src_len; 732 733 int src_idx, dst_idx, pl_idx; 734 size_t src_offset; 735 size_t fragment_offset = 0; 736 struct virtio_net_hdr virt_hdr = { 737 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ? 738 VIRTIO_NET_HDR_F_DATA_VALID : 0 739 }; 740 741 /* Copy headers */ 742 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr; 743 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr); 744 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 745 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 746 747 switch (gso_type) { 748 case VIRTIO_NET_HDR_GSO_TCPV4: 749 case VIRTIO_NET_HDR_GSO_TCPV6: 750 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len, 751 &src_idx, &src_offset, &src_len)) { 752 return false; 753 } 754 break; 755 756 case VIRTIO_NET_HDR_GSO_UDP: 757 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 758 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 759 pkt->payload_len); 760 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len, 761 &src_idx, &src_offset, &src_len); 762 break; 763 764 default: 765 abort(); 766 } 767 768 /* Put as much data as possible and send */ 769 while (true) { 770 dst_idx = pl_idx; 771 fragment_len = net_tx_pkt_fetch_fragment(pkt, 772 &src_idx, &src_offset, src_len, fragment, &dst_idx); 773 if (!fragment_len) { 774 break; 775 } 776 777 switch (gso_type) { 778 case VIRTIO_NET_HDR_GSO_TCPV4: 779 case VIRTIO_NET_HDR_GSO_TCPV6: 780 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type); 781 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG, 782 dst_idx - NET_TX_PKT_L2HDR_FRAG, 783 l4hdr_len + fragment_len); 784 break; 785 786 case VIRTIO_NET_HDR_GSO_UDP: 787 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset, 788 fragment_len); 789 break; 790 } 791 792 callback(context, 793 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG, 794 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG); 795 796 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 797 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 798 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len, 799 gso_type); 800 } 801 802 fragment_offset += fragment_len; 803 } 804 805 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 806 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 807 net_tx_pkt_tcp_fragment_deinit(fragment); 808 } 809 810 return true; 811 } 812 813 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) 814 { 815 bool offload = qemu_get_vnet_hdr_len(nc->peer); 816 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc); 817 } 818 819 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, 820 NetTxPktSend callback, void *context) 821 { 822 assert(pkt); 823 824 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 825 826 /* 827 * Since underlying infrastructure does not support IP datagrams longer 828 * than 64K we should drop such packets and don't even try to send 829 */ 830 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) { 831 if (pkt->payload_len > 832 ETH_MAX_IP_DGRAM_LEN - 833 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) { 834 return false; 835 } 836 } 837 838 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) { 839 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 840 pkt->virt_hdr.flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM; 841 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 842 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 843 pkt->payload_len); 844 } 845 846 net_tx_pkt_fix_ip6_payload_len(pkt); 847 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG, 848 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG, 849 pkt->vec + NET_TX_PKT_VHDR_FRAG, 850 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG); 851 return true; 852 } 853 854 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context); 855 } 856 857 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt) 858 { 859 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 860 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) { 861 /* 862 * TODO: if qemu would support >64K packets - add jumbo option check 863 * something like that: 864 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {' 865 */ 866 if (pkt->l3_hdr.ip6.ip6_plen == 0) { 867 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) { 868 pkt->l3_hdr.ip6.ip6_plen = htons(pkt->payload_len); 869 } 870 /* 871 * TODO: if qemu would support >64K packets 872 * add jumbo option for packets greater then 65,535 bytes 873 */ 874 } 875 } 876 } 877