1 /* 2 * QEMU TX packets abstractions 3 * 4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com) 5 * 6 * Developed by Daynix Computing LTD (http://www.daynix.com) 7 * 8 * Authors: 9 * Dmitry Fleytman <dmitry@daynix.com> 10 * Tamir Shomer <tamirs@daynix.com> 11 * Yan Vugenfirer <yan@daynix.com> 12 * 13 * This work is licensed under the terms of the GNU GPL, version 2 or later. 14 * See the COPYING file in the top-level directory. 15 * 16 */ 17 18 #include "qemu/osdep.h" 19 #include "net/eth.h" 20 #include "net/checksum.h" 21 #include "net/tap.h" 22 #include "net/net.h" 23 #include "hw/pci/pci_device.h" 24 #include "net_tx_pkt.h" 25 26 enum { 27 NET_TX_PKT_VHDR_FRAG = 0, 28 NET_TX_PKT_L2HDR_FRAG, 29 NET_TX_PKT_L3HDR_FRAG, 30 NET_TX_PKT_PL_START_FRAG 31 }; 32 33 /* TX packet private context */ 34 struct NetTxPkt { 35 struct virtio_net_hdr virt_hdr; 36 37 struct iovec *raw; 38 uint32_t raw_frags; 39 uint32_t max_raw_frags; 40 41 struct iovec *vec; 42 43 uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN]; 44 union { 45 struct ip_header ip; 46 struct ip6_header ip6; 47 uint8_t octets[ETH_MAX_IP_DGRAM_LEN]; 48 } l3_hdr; 49 50 uint32_t payload_len; 51 52 uint32_t payload_frags; 53 uint32_t max_payload_frags; 54 55 uint16_t hdr_len; 56 eth_pkt_types_e packet_type; 57 uint8_t l4proto; 58 }; 59 60 void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags) 61 { 62 struct NetTxPkt *p = g_malloc0(sizeof *p); 63 64 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG); 65 66 p->raw = g_new(struct iovec, max_frags); 67 68 p->max_payload_frags = max_frags; 69 p->max_raw_frags = max_frags; 70 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr; 71 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr; 72 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr; 73 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr; 74 75 *pkt = p; 76 } 77 78 void net_tx_pkt_uninit(struct NetTxPkt *pkt) 79 { 80 if (pkt) { 81 g_free(pkt->vec); 82 g_free(pkt->raw); 83 g_free(pkt); 84 } 85 } 86 87 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt) 88 { 89 uint16_t csum; 90 assert(pkt); 91 92 pkt->l3_hdr.ip.ip_len = cpu_to_be16(pkt->payload_len + 93 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 94 95 pkt->l3_hdr.ip.ip_sum = 0; 96 csum = net_raw_checksum(pkt->l3_hdr.octets, 97 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len); 98 pkt->l3_hdr.ip.ip_sum = cpu_to_be16(csum); 99 } 100 101 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt) 102 { 103 uint16_t csum; 104 uint32_t cntr, cso; 105 assert(pkt); 106 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 107 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base; 108 109 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len > 110 ETH_MAX_IP_DGRAM_LEN) { 111 return; 112 } 113 114 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 115 gso_type == VIRTIO_NET_HDR_GSO_UDP) { 116 /* Calculate IP header checksum */ 117 net_tx_pkt_update_ip_hdr_checksum(pkt); 118 119 /* Calculate IP pseudo header checksum */ 120 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso); 121 csum = cpu_to_be16(~net_checksum_finish(cntr)); 122 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 123 /* Calculate IP pseudo header checksum */ 124 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len, 125 IP_PROTO_TCP, &cso); 126 csum = cpu_to_be16(~net_checksum_finish(cntr)); 127 } else { 128 return; 129 } 130 131 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags, 132 pkt->virt_hdr.csum_offset, &csum, sizeof(csum)); 133 } 134 135 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt) 136 { 137 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len + 138 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len; 139 } 140 141 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt) 142 { 143 struct iovec *l2_hdr, *l3_hdr; 144 size_t bytes_read; 145 size_t full_ip6hdr_len; 146 uint16_t l3_proto; 147 148 assert(pkt); 149 150 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 151 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 152 153 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base, 154 ETH_MAX_L2_HDR_LEN); 155 if (bytes_read < sizeof(struct eth_header)) { 156 l2_hdr->iov_len = 0; 157 return false; 158 } 159 160 l2_hdr->iov_len = sizeof(struct eth_header); 161 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) { 162 case ETH_P_VLAN: 163 l2_hdr->iov_len += sizeof(struct vlan_header); 164 break; 165 case ETH_P_DVLAN: 166 l2_hdr->iov_len += 2 * sizeof(struct vlan_header); 167 break; 168 } 169 170 if (bytes_read < l2_hdr->iov_len) { 171 l2_hdr->iov_len = 0; 172 l3_hdr->iov_len = 0; 173 pkt->packet_type = ETH_PKT_UCAST; 174 return false; 175 } else { 176 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN; 177 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base); 178 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base); 179 } 180 181 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len); 182 183 switch (l3_proto) { 184 case ETH_P_IP: 185 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 186 l3_hdr->iov_base, sizeof(struct ip_header)); 187 188 if (bytes_read < sizeof(struct ip_header)) { 189 l3_hdr->iov_len = 0; 190 return false; 191 } 192 193 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base); 194 195 if (l3_hdr->iov_len < sizeof(struct ip_header)) { 196 l3_hdr->iov_len = 0; 197 return false; 198 } 199 200 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base); 201 202 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) { 203 /* copy optional IPv4 header data if any*/ 204 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 205 l2_hdr->iov_len + sizeof(struct ip_header), 206 l3_hdr->iov_base + sizeof(struct ip_header), 207 l3_hdr->iov_len - sizeof(struct ip_header)); 208 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) { 209 l3_hdr->iov_len = 0; 210 return false; 211 } 212 } 213 214 break; 215 216 case ETH_P_IPV6: 217 { 218 eth_ip6_hdr_info hdrinfo; 219 220 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 221 &hdrinfo)) { 222 l3_hdr->iov_len = 0; 223 return false; 224 } 225 226 pkt->l4proto = hdrinfo.l4proto; 227 full_ip6hdr_len = hdrinfo.full_hdr_len; 228 229 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) { 230 l3_hdr->iov_len = 0; 231 return false; 232 } 233 234 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len, 235 l3_hdr->iov_base, full_ip6hdr_len); 236 237 if (bytes_read < full_ip6hdr_len) { 238 l3_hdr->iov_len = 0; 239 return false; 240 } else { 241 l3_hdr->iov_len = full_ip6hdr_len; 242 } 243 break; 244 } 245 default: 246 l3_hdr->iov_len = 0; 247 break; 248 } 249 250 net_tx_pkt_calculate_hdr_len(pkt); 251 return true; 252 } 253 254 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt) 255 { 256 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len; 257 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 258 pkt->max_payload_frags, 259 pkt->raw, pkt->raw_frags, 260 pkt->hdr_len, pkt->payload_len); 261 } 262 263 bool net_tx_pkt_parse(struct NetTxPkt *pkt) 264 { 265 if (net_tx_pkt_parse_headers(pkt)) { 266 net_tx_pkt_rebuild_payload(pkt); 267 return true; 268 } else { 269 return false; 270 } 271 } 272 273 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt) 274 { 275 assert(pkt); 276 return &pkt->virt_hdr; 277 } 278 279 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt, 280 bool tso_enable) 281 { 282 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE; 283 uint16_t l3_proto; 284 285 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1, 286 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len); 287 288 if (!tso_enable) { 289 goto func_exit; 290 } 291 292 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 293 pkt->l4proto); 294 295 func_exit: 296 return rc; 297 } 298 299 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable, 300 bool csum_enable, uint32_t gso_size) 301 { 302 struct tcp_hdr l4hdr; 303 size_t bytes_read; 304 assert(pkt); 305 306 /* csum has to be enabled if tso is. */ 307 assert(csum_enable || !tso_enable); 308 309 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable); 310 311 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 312 case VIRTIO_NET_HDR_GSO_NONE: 313 pkt->virt_hdr.hdr_len = 0; 314 pkt->virt_hdr.gso_size = 0; 315 break; 316 317 case VIRTIO_NET_HDR_GSO_UDP: 318 pkt->virt_hdr.gso_size = gso_size; 319 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header); 320 break; 321 322 case VIRTIO_NET_HDR_GSO_TCPV4: 323 case VIRTIO_NET_HDR_GSO_TCPV6: 324 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], 325 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr)); 326 if (bytes_read < sizeof(l4hdr) || 327 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) { 328 return false; 329 } 330 331 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t); 332 pkt->virt_hdr.gso_size = gso_size; 333 break; 334 335 default: 336 g_assert_not_reached(); 337 } 338 339 if (csum_enable) { 340 switch (pkt->l4proto) { 341 case IP_PROTO_TCP: 342 if (pkt->payload_len < sizeof(struct tcp_hdr)) { 343 return false; 344 } 345 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 346 pkt->virt_hdr.csum_start = pkt->hdr_len; 347 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum); 348 break; 349 case IP_PROTO_UDP: 350 if (pkt->payload_len < sizeof(struct udp_hdr)) { 351 return false; 352 } 353 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 354 pkt->virt_hdr.csum_start = pkt->hdr_len; 355 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum); 356 break; 357 default: 358 break; 359 } 360 } 361 362 return true; 363 } 364 365 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt, 366 uint16_t vlan, uint16_t vlan_ethtype) 367 { 368 bool is_new; 369 assert(pkt); 370 371 eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base, 372 vlan, vlan_ethtype, &is_new); 373 374 /* update l2hdrlen */ 375 if (is_new) { 376 pkt->hdr_len += sizeof(struct vlan_header); 377 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len += 378 sizeof(struct vlan_header); 379 } 380 } 381 382 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len) 383 { 384 struct iovec *ventry; 385 assert(pkt); 386 387 if (pkt->raw_frags >= pkt->max_raw_frags) { 388 return false; 389 } 390 391 ventry = &pkt->raw[pkt->raw_frags]; 392 ventry->iov_base = base; 393 ventry->iov_len = len; 394 pkt->raw_frags++; 395 396 return true; 397 } 398 399 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt) 400 { 401 return pkt->raw_frags > 0; 402 } 403 404 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt) 405 { 406 assert(pkt); 407 408 return pkt->packet_type; 409 } 410 411 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt) 412 { 413 assert(pkt); 414 415 return pkt->hdr_len + pkt->payload_len; 416 } 417 418 void net_tx_pkt_dump(struct NetTxPkt *pkt) 419 { 420 #ifdef NET_TX_PKT_DEBUG 421 assert(pkt); 422 423 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, " 424 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type, 425 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len, 426 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len); 427 #endif 428 } 429 430 void net_tx_pkt_reset(struct NetTxPkt *pkt, 431 NetTxPktFreeFrag callback, void *context) 432 { 433 int i; 434 435 /* no assert, as reset can be called before tx_pkt_init */ 436 if (!pkt) { 437 return; 438 } 439 440 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr)); 441 442 assert(pkt->vec); 443 444 pkt->payload_len = 0; 445 pkt->payload_frags = 0; 446 447 if (pkt->max_raw_frags > 0) { 448 assert(pkt->raw); 449 for (i = 0; i < pkt->raw_frags; i++) { 450 assert(pkt->raw[i].iov_base); 451 callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len); 452 } 453 } 454 pkt->raw_frags = 0; 455 456 pkt->hdr_len = 0; 457 pkt->l4proto = 0; 458 } 459 460 void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len) 461 { 462 pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0); 463 } 464 465 bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev, 466 dma_addr_t pa, size_t len) 467 { 468 dma_addr_t mapped_len = len; 469 void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE); 470 if (!base) { 471 return false; 472 } 473 474 if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) { 475 net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len); 476 return false; 477 } 478 479 return true; 480 } 481 482 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt, 483 struct iovec *iov, uint32_t iov_len, 484 uint16_t csl) 485 { 486 uint32_t csum_cntr; 487 uint16_t csum = 0; 488 uint32_t cso; 489 /* num of iovec without vhdr */ 490 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset; 491 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len); 492 493 /* Put zero to checksum field */ 494 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 495 496 /* Calculate L4 TCP/UDP checksum */ 497 csum_cntr = 0; 498 cso = 0; 499 /* add pseudo header to csum */ 500 if (l3_proto == ETH_P_IP) { 501 csum_cntr = eth_calc_ip4_pseudo_hdr_csum( 502 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 503 csl, &cso); 504 } else if (l3_proto == ETH_P_IPV6) { 505 csum_cntr = eth_calc_ip6_pseudo_hdr_csum( 506 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base, 507 csl, pkt->l4proto, &cso); 508 } 509 510 /* data checksum */ 511 csum_cntr += 512 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso); 513 514 /* Put the checksum obtained into the packet */ 515 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr)); 516 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum); 517 } 518 519 #define NET_MAX_FRAG_SG_LIST (64) 520 521 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt, 522 int *src_idx, size_t *src_offset, size_t src_len, 523 struct iovec *dst, int *dst_idx) 524 { 525 size_t fetched = 0; 526 struct iovec *src = pkt->vec; 527 528 while (fetched < src_len) { 529 530 /* no more place in fragment iov */ 531 if (*dst_idx == NET_MAX_FRAG_SG_LIST) { 532 break; 533 } 534 535 /* no more data in iovec */ 536 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) { 537 break; 538 } 539 540 541 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset; 542 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset, 543 src_len - fetched); 544 545 *src_offset += dst[*dst_idx].iov_len; 546 fetched += dst[*dst_idx].iov_len; 547 548 if (*src_offset == src[*src_idx].iov_len) { 549 *src_offset = 0; 550 (*src_idx)++; 551 } 552 553 (*dst_idx)++; 554 } 555 556 return fetched; 557 } 558 559 static void net_tx_pkt_sendv( 560 void *opaque, const struct iovec *iov, int iov_cnt, 561 const struct iovec *virt_iov, int virt_iov_cnt) 562 { 563 NetClientState *nc = opaque; 564 565 if (qemu_get_using_vnet_hdr(nc->peer)) { 566 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt); 567 } else { 568 qemu_sendv_packet(nc, iov, iov_cnt); 569 } 570 } 571 572 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt, 573 struct iovec *fragment, 574 int *pl_idx, 575 size_t *l4hdr_len, 576 int *src_idx, 577 size_t *src_offset, 578 size_t *src_len) 579 { 580 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG; 581 size_t bytes_read = 0; 582 struct tcp_hdr *th; 583 584 if (!pkt->payload_frags) { 585 return false; 586 } 587 588 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len; 589 l4->iov_base = g_malloc(l4->iov_len); 590 591 *src_idx = NET_TX_PKT_PL_START_FRAG; 592 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) { 593 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 594 pkt->vec[*src_idx].iov_len); 595 596 bytes_read += pkt->vec[*src_idx].iov_len; 597 598 (*src_idx)++; 599 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) { 600 g_free(l4->iov_base); 601 return false; 602 } 603 } 604 605 *src_offset = l4->iov_len - bytes_read; 606 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base, 607 *src_offset); 608 609 th = l4->iov_base; 610 th->th_flags &= ~(TH_FIN | TH_PUSH); 611 612 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1; 613 *l4hdr_len = l4->iov_len; 614 *src_len = pkt->virt_hdr.gso_size; 615 616 return true; 617 } 618 619 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment) 620 { 621 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base); 622 } 623 624 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt, 625 struct iovec *fragment, 626 size_t fragment_len, 627 uint8_t gso_type) 628 { 629 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 630 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 631 struct ip_header *ip = l3hdr->iov_base; 632 struct ip6_header *ip6 = l3hdr->iov_base; 633 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len; 634 635 switch (gso_type) { 636 case VIRTIO_NET_HDR_GSO_TCPV4: 637 ip->ip_len = cpu_to_be16(len); 638 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 639 break; 640 641 case VIRTIO_NET_HDR_GSO_TCPV6: 642 len -= sizeof(struct ip6_header); 643 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len); 644 break; 645 } 646 } 647 648 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt, 649 struct iovec *fragment, 650 size_t fragment_len, 651 uint8_t gso_type) 652 { 653 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 654 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG; 655 struct ip_header *ip = l3hdr->iov_base; 656 struct tcp_hdr *th = l4hdr->iov_base; 657 658 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) { 659 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1); 660 } 661 662 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len); 663 th->th_flags &= ~TH_CWR; 664 } 665 666 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt, 667 int *pl_idx, 668 size_t *l4hdr_len, 669 int *src_idx, size_t *src_offset, 670 size_t *src_len) 671 { 672 *pl_idx = NET_TX_PKT_PL_START_FRAG; 673 *l4hdr_len = 0; 674 *src_idx = NET_TX_PKT_PL_START_FRAG; 675 *src_offset = 0; 676 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size); 677 } 678 679 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt, 680 struct iovec *fragment, 681 size_t fragment_offset, 682 size_t fragment_len) 683 { 684 bool more_frags = fragment_offset + fragment_len < pkt->payload_len; 685 uint16_t orig_flags; 686 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG; 687 struct ip_header *ip = l3hdr->iov_base; 688 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE; 689 uint16_t new_ip_off; 690 691 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0); 692 assert((frag_off_units & ~IP_OFFMASK) == 0); 693 694 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF); 695 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0); 696 ip->ip_off = cpu_to_be16(new_ip_off); 697 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len); 698 699 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len); 700 } 701 702 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt, 703 NetTxPktSend callback, 704 void *context) 705 { 706 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 707 708 struct iovec fragment[NET_MAX_FRAG_SG_LIST]; 709 size_t fragment_len; 710 size_t l4hdr_len; 711 size_t src_len; 712 713 int src_idx, dst_idx, pl_idx; 714 size_t src_offset; 715 size_t fragment_offset = 0; 716 struct virtio_net_hdr virt_hdr = { 717 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ? 718 VIRTIO_NET_HDR_F_DATA_VALID : 0 719 }; 720 721 /* Copy headers */ 722 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr; 723 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr); 724 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 725 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG]; 726 727 switch (gso_type) { 728 case VIRTIO_NET_HDR_GSO_TCPV4: 729 case VIRTIO_NET_HDR_GSO_TCPV6: 730 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len, 731 &src_idx, &src_offset, &src_len)) { 732 return false; 733 } 734 break; 735 736 case VIRTIO_NET_HDR_GSO_UDP: 737 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 738 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 739 pkt->payload_len); 740 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len, 741 &src_idx, &src_offset, &src_len); 742 break; 743 744 default: 745 abort(); 746 } 747 748 /* Put as much data as possible and send */ 749 while (true) { 750 dst_idx = pl_idx; 751 fragment_len = net_tx_pkt_fetch_fragment(pkt, 752 &src_idx, &src_offset, src_len, fragment, &dst_idx); 753 if (!fragment_len) { 754 break; 755 } 756 757 switch (gso_type) { 758 case VIRTIO_NET_HDR_GSO_TCPV4: 759 case VIRTIO_NET_HDR_GSO_TCPV6: 760 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type); 761 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG, 762 dst_idx - NET_TX_PKT_L2HDR_FRAG, 763 l4hdr_len + fragment_len); 764 break; 765 766 case VIRTIO_NET_HDR_GSO_UDP: 767 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset, 768 fragment_len); 769 break; 770 } 771 772 callback(context, 773 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG, 774 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG); 775 776 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 777 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 778 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len, 779 gso_type); 780 } 781 782 fragment_offset += fragment_len; 783 } 784 785 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 || 786 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) { 787 net_tx_pkt_tcp_fragment_deinit(fragment); 788 } 789 790 return true; 791 } 792 793 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc) 794 { 795 bool offload = qemu_get_using_vnet_hdr(nc->peer); 796 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc); 797 } 798 799 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload, 800 NetTxPktSend callback, void *context) 801 { 802 assert(pkt); 803 804 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN; 805 806 /* 807 * Since underlying infrastructure does not support IP datagrams longer 808 * than 64K we should drop such packets and don't even try to send 809 */ 810 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) { 811 if (pkt->payload_len > 812 ETH_MAX_IP_DGRAM_LEN - 813 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) { 814 return false; 815 } 816 } 817 818 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) { 819 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 820 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG], 821 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1, 822 pkt->payload_len); 823 } 824 825 net_tx_pkt_fix_ip6_payload_len(pkt); 826 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG, 827 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG, 828 pkt->vec + NET_TX_PKT_VHDR_FRAG, 829 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG); 830 return true; 831 } 832 833 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context); 834 } 835 836 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt) 837 { 838 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG]; 839 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) { 840 /* 841 * TODO: if qemu would support >64K packets - add jumbo option check 842 * something like that: 843 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {' 844 */ 845 if (pkt->l3_hdr.ip6.ip6_plen == 0) { 846 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) { 847 pkt->l3_hdr.ip6.ip6_plen = htons(pkt->payload_len); 848 } 849 /* 850 * TODO: if qemu would support >64K packets 851 * add jumbo option for packets greater then 65,535 bytes 852 */ 853 } 854 } 855 } 856