1 /*
2 * QEMU TX packets abstractions
3 *
4 * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
5 *
6 * Developed by Daynix Computing LTD (http://www.daynix.com)
7 *
8 * Authors:
9 * Dmitry Fleytman <dmitry@daynix.com>
10 * Tamir Shomer <tamirs@daynix.com>
11 * Yan Vugenfirer <yan@daynix.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2 or later.
14 * See the COPYING file in the top-level directory.
15 *
16 */
17
18 #include "qemu/osdep.h"
19 #include "qemu/crc32c.h"
20 #include "net/eth.h"
21 #include "net/checksum.h"
22 #include "net/tap.h"
23 #include "net/net.h"
24 #include "hw/pci/pci_device.h"
25 #include "net_tx_pkt.h"
26
27 enum {
28 NET_TX_PKT_VHDR_FRAG = 0,
29 NET_TX_PKT_L2HDR_FRAG,
30 NET_TX_PKT_L3HDR_FRAG,
31 NET_TX_PKT_PL_START_FRAG
32 };
33
34 /* TX packet private context */
35 struct NetTxPkt {
36 struct virtio_net_hdr virt_hdr;
37
38 struct iovec *raw;
39 uint32_t raw_frags;
40 uint32_t max_raw_frags;
41
42 struct iovec *vec;
43
44 struct {
45 struct eth_header eth;
46 struct vlan_header vlan[3];
47 } l2_hdr;
48 union {
49 struct ip_header ip;
50 struct ip6_header ip6;
51 uint8_t octets[ETH_MAX_IP_DGRAM_LEN];
52 } l3_hdr;
53
54 uint32_t payload_len;
55
56 uint32_t payload_frags;
57 uint32_t max_payload_frags;
58
59 uint16_t hdr_len;
60 eth_pkt_types_e packet_type;
61 uint8_t l4proto;
62 };
63
net_tx_pkt_init(struct NetTxPkt ** pkt,uint32_t max_frags)64 void net_tx_pkt_init(struct NetTxPkt **pkt, uint32_t max_frags)
65 {
66 struct NetTxPkt *p = g_malloc0(sizeof *p);
67
68 p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
69
70 p->raw = g_new(struct iovec, max_frags);
71
72 p->max_payload_frags = max_frags;
73 p->max_raw_frags = max_frags;
74 p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
75 p->vec[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof p->virt_hdr;
76 p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
77 p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr;
78
79 *pkt = p;
80 }
81
net_tx_pkt_uninit(struct NetTxPkt * pkt)82 void net_tx_pkt_uninit(struct NetTxPkt *pkt)
83 {
84 if (pkt) {
85 g_free(pkt->vec);
86 g_free(pkt->raw);
87 g_free(pkt);
88 }
89 }
90
net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt * pkt)91 void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt)
92 {
93 uint16_t csum;
94 assert(pkt);
95
96 pkt->l3_hdr.ip.ip_len = cpu_to_be16(pkt->payload_len +
97 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
98
99 pkt->l3_hdr.ip.ip_sum = 0;
100 csum = net_raw_checksum(pkt->l3_hdr.octets,
101 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
102 pkt->l3_hdr.ip.ip_sum = cpu_to_be16(csum);
103 }
104
net_tx_pkt_update_ip_checksums(struct NetTxPkt * pkt)105 void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt)
106 {
107 uint16_t csum;
108 uint32_t cntr, cso;
109 assert(pkt);
110 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
111 void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
112
113 if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len >
114 ETH_MAX_IP_DGRAM_LEN) {
115 return;
116 }
117
118 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
119 gso_type == VIRTIO_NET_HDR_GSO_UDP) {
120 /* Calculate IP header checksum */
121 net_tx_pkt_update_ip_hdr_checksum(pkt);
122
123 /* Calculate IP pseudo header checksum */
124 cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso);
125 csum = cpu_to_be16(~net_checksum_finish(cntr));
126 } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
127 /* Calculate IP pseudo header checksum */
128 cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len,
129 IP_PROTO_TCP, &cso);
130 csum = cpu_to_be16(~net_checksum_finish(cntr));
131 } else {
132 return;
133 }
134
135 iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
136 pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
137 }
138
net_tx_pkt_update_sctp_checksum(struct NetTxPkt * pkt)139 bool net_tx_pkt_update_sctp_checksum(struct NetTxPkt *pkt)
140 {
141 uint32_t csum = 0;
142 struct iovec *pl_start_frag = pkt->vec + NET_TX_PKT_PL_START_FRAG;
143
144 if (iov_size(pl_start_frag, pkt->payload_frags) < 8 + sizeof(csum)) {
145 return false;
146 }
147
148 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
149 return false;
150 }
151
152 csum = cpu_to_le32(iov_crc32c(0xffffffff, pl_start_frag, pkt->payload_frags));
153 if (iov_from_buf(pl_start_frag, pkt->payload_frags, 8, &csum, sizeof(csum)) < sizeof(csum)) {
154 return false;
155 }
156
157 return true;
158 }
159
net_tx_pkt_calculate_hdr_len(struct NetTxPkt * pkt)160 static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
161 {
162 pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
163 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
164 }
165
net_tx_pkt_parse_headers(struct NetTxPkt * pkt)166 static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt)
167 {
168 struct iovec *l2_hdr, *l3_hdr;
169 size_t bytes_read;
170 size_t full_ip6hdr_len;
171 uint16_t l3_proto;
172
173 assert(pkt);
174
175 l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
176 l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG];
177
178 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
179 ETH_MAX_L2_HDR_LEN);
180 if (bytes_read < sizeof(struct eth_header)) {
181 l2_hdr->iov_len = 0;
182 return false;
183 }
184
185 l2_hdr->iov_len = sizeof(struct eth_header);
186 switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
187 case ETH_P_VLAN:
188 l2_hdr->iov_len += sizeof(struct vlan_header);
189 break;
190 case ETH_P_DVLAN:
191 l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
192 break;
193 }
194
195 if (bytes_read < l2_hdr->iov_len) {
196 l2_hdr->iov_len = 0;
197 l3_hdr->iov_len = 0;
198 pkt->packet_type = ETH_PKT_UCAST;
199 return false;
200 } else {
201 l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN;
202 l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
203 pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
204 }
205
206 l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len);
207
208 switch (l3_proto) {
209 case ETH_P_IP:
210 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
211 l3_hdr->iov_base, sizeof(struct ip_header));
212
213 if (bytes_read < sizeof(struct ip_header)) {
214 l3_hdr->iov_len = 0;
215 return false;
216 }
217
218 l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
219
220 if (l3_hdr->iov_len < sizeof(struct ip_header)) {
221 l3_hdr->iov_len = 0;
222 return false;
223 }
224
225 pkt->l4proto = IP_HDR_GET_P(l3_hdr->iov_base);
226
227 if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) {
228 /* copy optional IPv4 header data if any*/
229 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
230 l2_hdr->iov_len + sizeof(struct ip_header),
231 l3_hdr->iov_base + sizeof(struct ip_header),
232 l3_hdr->iov_len - sizeof(struct ip_header));
233 if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
234 l3_hdr->iov_len = 0;
235 return false;
236 }
237 }
238
239 break;
240
241 case ETH_P_IPV6:
242 {
243 eth_ip6_hdr_info hdrinfo;
244
245 if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
246 &hdrinfo)) {
247 l3_hdr->iov_len = 0;
248 return false;
249 }
250
251 pkt->l4proto = hdrinfo.l4proto;
252 full_ip6hdr_len = hdrinfo.full_hdr_len;
253
254 if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) {
255 l3_hdr->iov_len = 0;
256 return false;
257 }
258
259 bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
260 l3_hdr->iov_base, full_ip6hdr_len);
261
262 if (bytes_read < full_ip6hdr_len) {
263 l3_hdr->iov_len = 0;
264 return false;
265 } else {
266 l3_hdr->iov_len = full_ip6hdr_len;
267 }
268 break;
269 }
270 default:
271 l3_hdr->iov_len = 0;
272 break;
273 }
274
275 net_tx_pkt_calculate_hdr_len(pkt);
276 return true;
277 }
278
net_tx_pkt_rebuild_payload(struct NetTxPkt * pkt)279 static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt)
280 {
281 pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
282 pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
283 pkt->max_payload_frags,
284 pkt->raw, pkt->raw_frags,
285 pkt->hdr_len, pkt->payload_len);
286 }
287
net_tx_pkt_parse(struct NetTxPkt * pkt)288 bool net_tx_pkt_parse(struct NetTxPkt *pkt)
289 {
290 if (net_tx_pkt_parse_headers(pkt)) {
291 net_tx_pkt_rebuild_payload(pkt);
292 return true;
293 } else {
294 return false;
295 }
296 }
297
net_tx_pkt_get_vhdr(struct NetTxPkt * pkt)298 struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt)
299 {
300 assert(pkt);
301 return &pkt->virt_hdr;
302 }
303
net_tx_pkt_get_gso_type(struct NetTxPkt * pkt,bool tso_enable)304 static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt,
305 bool tso_enable)
306 {
307 uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
308 uint16_t l3_proto;
309
310 l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1,
311 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len);
312
313 if (!tso_enable) {
314 goto func_exit;
315 }
316
317 rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
318 pkt->l4proto);
319
320 func_exit:
321 return rc;
322 }
323
net_tx_pkt_build_vheader(struct NetTxPkt * pkt,bool tso_enable,bool csum_enable,uint32_t gso_size)324 bool net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
325 bool csum_enable, uint32_t gso_size)
326 {
327 struct tcp_hdr l4hdr;
328 size_t bytes_read;
329 assert(pkt);
330
331 /* csum has to be enabled if tso is. */
332 assert(csum_enable || !tso_enable);
333
334 pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable);
335
336 switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
337 case VIRTIO_NET_HDR_GSO_NONE:
338 pkt->virt_hdr.hdr_len = 0;
339 pkt->virt_hdr.gso_size = 0;
340 break;
341
342 case VIRTIO_NET_HDR_GSO_UDP:
343 pkt->virt_hdr.gso_size = gso_size;
344 pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
345 break;
346
347 case VIRTIO_NET_HDR_GSO_TCPV4:
348 case VIRTIO_NET_HDR_GSO_TCPV6:
349 bytes_read = iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
350 pkt->payload_frags, 0, &l4hdr, sizeof(l4hdr));
351 if (bytes_read < sizeof(l4hdr) ||
352 l4hdr.th_off * sizeof(uint32_t) < sizeof(l4hdr)) {
353 return false;
354 }
355
356 pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
357 pkt->virt_hdr.gso_size = gso_size;
358 break;
359
360 default:
361 g_assert_not_reached();
362 }
363
364 if (csum_enable) {
365 switch (pkt->l4proto) {
366 case IP_PROTO_TCP:
367 if (pkt->payload_len < sizeof(struct tcp_hdr)) {
368 return false;
369 }
370 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
371 pkt->virt_hdr.csum_start = pkt->hdr_len;
372 pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
373 break;
374 case IP_PROTO_UDP:
375 if (pkt->payload_len < sizeof(struct udp_hdr)) {
376 return false;
377 }
378 pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
379 pkt->virt_hdr.csum_start = pkt->hdr_len;
380 pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
381 break;
382 default:
383 break;
384 }
385 }
386
387 return true;
388 }
389
net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt * pkt,uint16_t vlan,uint16_t vlan_ethtype)390 void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
391 uint16_t vlan, uint16_t vlan_ethtype)
392 {
393 assert(pkt);
394
395 eth_setup_vlan_headers(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
396 &pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
397 vlan, vlan_ethtype);
398
399 pkt->hdr_len += sizeof(struct vlan_header);
400 }
401
net_tx_pkt_add_raw_fragment(struct NetTxPkt * pkt,void * base,size_t len)402 bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, void *base, size_t len)
403 {
404 struct iovec *ventry;
405 assert(pkt);
406
407 if (pkt->raw_frags >= pkt->max_raw_frags) {
408 return false;
409 }
410
411 ventry = &pkt->raw[pkt->raw_frags];
412 ventry->iov_base = base;
413 ventry->iov_len = len;
414 pkt->raw_frags++;
415
416 return true;
417 }
418
net_tx_pkt_has_fragments(struct NetTxPkt * pkt)419 bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
420 {
421 return pkt->raw_frags > 0;
422 }
423
net_tx_pkt_get_packet_type(struct NetTxPkt * pkt)424 eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt)
425 {
426 assert(pkt);
427
428 return pkt->packet_type;
429 }
430
net_tx_pkt_get_total_len(struct NetTxPkt * pkt)431 size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt)
432 {
433 assert(pkt);
434
435 return pkt->hdr_len + pkt->payload_len;
436 }
437
net_tx_pkt_dump(struct NetTxPkt * pkt)438 void net_tx_pkt_dump(struct NetTxPkt *pkt)
439 {
440 #ifdef NET_TX_PKT_DEBUG
441 assert(pkt);
442
443 printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
444 "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
445 pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
446 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
447 #endif
448 }
449
net_tx_pkt_reset(struct NetTxPkt * pkt,NetTxPktFreeFrag callback,void * context)450 void net_tx_pkt_reset(struct NetTxPkt *pkt,
451 NetTxPktFreeFrag callback, void *context)
452 {
453 int i;
454
455 /* no assert, as reset can be called before tx_pkt_init */
456 if (!pkt) {
457 return;
458 }
459
460 memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
461
462 assert(pkt->vec);
463
464 pkt->payload_len = 0;
465 pkt->payload_frags = 0;
466
467 if (pkt->max_raw_frags > 0) {
468 assert(pkt->raw);
469 for (i = 0; i < pkt->raw_frags; i++) {
470 assert(pkt->raw[i].iov_base);
471 callback(context, pkt->raw[i].iov_base, pkt->raw[i].iov_len);
472 }
473 }
474 pkt->raw_frags = 0;
475
476 pkt->hdr_len = 0;
477 pkt->l4proto = 0;
478 }
479
net_tx_pkt_unmap_frag_pci(void * context,void * base,size_t len)480 void net_tx_pkt_unmap_frag_pci(void *context, void *base, size_t len)
481 {
482 pci_dma_unmap(context, base, len, DMA_DIRECTION_TO_DEVICE, 0);
483 }
484
net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt * pkt,PCIDevice * pci_dev,dma_addr_t pa,size_t len)485 bool net_tx_pkt_add_raw_fragment_pci(struct NetTxPkt *pkt, PCIDevice *pci_dev,
486 dma_addr_t pa, size_t len)
487 {
488 dma_addr_t mapped_len = len;
489 void *base = pci_dma_map(pci_dev, pa, &mapped_len, DMA_DIRECTION_TO_DEVICE);
490 if (!base) {
491 return false;
492 }
493
494 if (mapped_len != len || !net_tx_pkt_add_raw_fragment(pkt, base, len)) {
495 net_tx_pkt_unmap_frag_pci(pci_dev, base, mapped_len);
496 return false;
497 }
498
499 return true;
500 }
501
net_tx_pkt_do_sw_csum(struct NetTxPkt * pkt,struct iovec * iov,uint32_t iov_len,uint16_t csl)502 static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt,
503 struct iovec *iov, uint32_t iov_len,
504 uint16_t csl)
505 {
506 uint32_t csum_cntr;
507 uint16_t csum = 0;
508 uint32_t cso;
509 /* num of iovec without vhdr */
510 size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
511 uint16_t l3_proto = eth_get_l3_proto(iov, 1, iov->iov_len);
512
513 /* Put zero to checksum field */
514 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
515
516 /* Calculate L4 TCP/UDP checksum */
517 csum_cntr = 0;
518 cso = 0;
519 /* add pseudo header to csum */
520 if (l3_proto == ETH_P_IP) {
521 csum_cntr = eth_calc_ip4_pseudo_hdr_csum(
522 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
523 csl, &cso);
524 } else if (l3_proto == ETH_P_IPV6) {
525 csum_cntr = eth_calc_ip6_pseudo_hdr_csum(
526 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
527 csl, pkt->l4proto, &cso);
528 }
529
530 /* data checksum */
531 csum_cntr +=
532 net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso);
533
534 /* Put the checksum obtained into the packet */
535 csum = cpu_to_be16(net_checksum_finish_nozero(csum_cntr));
536 iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
537 }
538
539 #define NET_MAX_FRAG_SG_LIST (64)
540
net_tx_pkt_fetch_fragment(struct NetTxPkt * pkt,int * src_idx,size_t * src_offset,size_t src_len,struct iovec * dst,int * dst_idx)541 static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt,
542 int *src_idx, size_t *src_offset, size_t src_len,
543 struct iovec *dst, int *dst_idx)
544 {
545 size_t fetched = 0;
546 struct iovec *src = pkt->vec;
547
548 while (fetched < src_len) {
549
550 /* no more place in fragment iov */
551 if (*dst_idx == NET_MAX_FRAG_SG_LIST) {
552 break;
553 }
554
555 /* no more data in iovec */
556 if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) {
557 break;
558 }
559
560
561 dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
562 dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
563 src_len - fetched);
564
565 *src_offset += dst[*dst_idx].iov_len;
566 fetched += dst[*dst_idx].iov_len;
567
568 if (*src_offset == src[*src_idx].iov_len) {
569 *src_offset = 0;
570 (*src_idx)++;
571 }
572
573 (*dst_idx)++;
574 }
575
576 return fetched;
577 }
578
net_tx_pkt_sendv(void * opaque,const struct iovec * iov,int iov_cnt,const struct iovec * virt_iov,int virt_iov_cnt)579 static void net_tx_pkt_sendv(
580 void *opaque, const struct iovec *iov, int iov_cnt,
581 const struct iovec *virt_iov, int virt_iov_cnt)
582 {
583 NetClientState *nc = opaque;
584
585 if (qemu_get_vnet_hdr_len(nc->peer)) {
586 qemu_sendv_packet(nc, virt_iov, virt_iov_cnt);
587 } else {
588 qemu_sendv_packet(nc, iov, iov_cnt);
589 }
590 }
591
net_tx_pkt_tcp_fragment_init(struct NetTxPkt * pkt,struct iovec * fragment,int * pl_idx,size_t * l4hdr_len,int * src_idx,size_t * src_offset,size_t * src_len)592 static bool net_tx_pkt_tcp_fragment_init(struct NetTxPkt *pkt,
593 struct iovec *fragment,
594 int *pl_idx,
595 size_t *l4hdr_len,
596 int *src_idx,
597 size_t *src_offset,
598 size_t *src_len)
599 {
600 struct iovec *l4 = fragment + NET_TX_PKT_PL_START_FRAG;
601 size_t bytes_read = 0;
602 struct tcp_hdr *th;
603
604 if (!pkt->payload_frags) {
605 return false;
606 }
607
608 l4->iov_len = pkt->virt_hdr.hdr_len - pkt->hdr_len;
609 l4->iov_base = g_malloc(l4->iov_len);
610
611 *src_idx = NET_TX_PKT_PL_START_FRAG;
612 while (pkt->vec[*src_idx].iov_len < l4->iov_len - bytes_read) {
613 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base,
614 pkt->vec[*src_idx].iov_len);
615
616 bytes_read += pkt->vec[*src_idx].iov_len;
617
618 (*src_idx)++;
619 if (*src_idx >= pkt->payload_frags + NET_TX_PKT_PL_START_FRAG) {
620 g_free(l4->iov_base);
621 return false;
622 }
623 }
624
625 *src_offset = l4->iov_len - bytes_read;
626 memcpy((char *)l4->iov_base + bytes_read, pkt->vec[*src_idx].iov_base,
627 *src_offset);
628
629 th = l4->iov_base;
630 th->th_flags &= ~(TH_FIN | TH_PUSH);
631
632 *pl_idx = NET_TX_PKT_PL_START_FRAG + 1;
633 *l4hdr_len = l4->iov_len;
634 *src_len = pkt->virt_hdr.gso_size;
635
636 return true;
637 }
638
net_tx_pkt_tcp_fragment_deinit(struct iovec * fragment)639 static void net_tx_pkt_tcp_fragment_deinit(struct iovec *fragment)
640 {
641 g_free(fragment[NET_TX_PKT_PL_START_FRAG].iov_base);
642 }
643
net_tx_pkt_tcp_fragment_fix(struct NetTxPkt * pkt,struct iovec * fragment,size_t fragment_len,uint8_t gso_type)644 static void net_tx_pkt_tcp_fragment_fix(struct NetTxPkt *pkt,
645 struct iovec *fragment,
646 size_t fragment_len,
647 uint8_t gso_type)
648 {
649 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
650 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG;
651 struct ip_header *ip = l3hdr->iov_base;
652 struct ip6_header *ip6 = l3hdr->iov_base;
653 size_t len = l3hdr->iov_len + l4hdr->iov_len + fragment_len;
654
655 switch (gso_type) {
656 case VIRTIO_NET_HDR_GSO_TCPV4:
657 ip->ip_len = cpu_to_be16(len);
658 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len);
659 break;
660
661 case VIRTIO_NET_HDR_GSO_TCPV6:
662 len -= sizeof(struct ip6_header);
663 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = cpu_to_be16(len);
664 break;
665 }
666 }
667
net_tx_pkt_tcp_fragment_advance(struct NetTxPkt * pkt,struct iovec * fragment,size_t fragment_len,uint8_t gso_type)668 static void net_tx_pkt_tcp_fragment_advance(struct NetTxPkt *pkt,
669 struct iovec *fragment,
670 size_t fragment_len,
671 uint8_t gso_type)
672 {
673 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
674 struct iovec *l4hdr = fragment + NET_TX_PKT_PL_START_FRAG;
675 struct ip_header *ip = l3hdr->iov_base;
676 struct tcp_hdr *th = l4hdr->iov_base;
677
678 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4) {
679 ip->ip_id = cpu_to_be16(be16_to_cpu(ip->ip_id) + 1);
680 }
681
682 th->th_seq = cpu_to_be32(be32_to_cpu(th->th_seq) + fragment_len);
683 th->th_flags &= ~TH_CWR;
684 }
685
net_tx_pkt_udp_fragment_init(struct NetTxPkt * pkt,int * pl_idx,size_t * l4hdr_len,int * src_idx,size_t * src_offset,size_t * src_len)686 static void net_tx_pkt_udp_fragment_init(struct NetTxPkt *pkt,
687 int *pl_idx,
688 size_t *l4hdr_len,
689 int *src_idx, size_t *src_offset,
690 size_t *src_len)
691 {
692 *pl_idx = NET_TX_PKT_PL_START_FRAG;
693 *l4hdr_len = 0;
694 *src_idx = NET_TX_PKT_PL_START_FRAG;
695 *src_offset = 0;
696 *src_len = IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size);
697 }
698
net_tx_pkt_udp_fragment_fix(struct NetTxPkt * pkt,struct iovec * fragment,size_t fragment_offset,size_t fragment_len)699 static void net_tx_pkt_udp_fragment_fix(struct NetTxPkt *pkt,
700 struct iovec *fragment,
701 size_t fragment_offset,
702 size_t fragment_len)
703 {
704 bool more_frags = fragment_offset + fragment_len < pkt->payload_len;
705 uint16_t orig_flags;
706 struct iovec *l3hdr = fragment + NET_TX_PKT_L3HDR_FRAG;
707 struct ip_header *ip = l3hdr->iov_base;
708 uint16_t frag_off_units = fragment_offset / IP_FRAG_UNIT_SIZE;
709 uint16_t new_ip_off;
710
711 assert(fragment_offset % IP_FRAG_UNIT_SIZE == 0);
712 assert((frag_off_units & ~IP_OFFMASK) == 0);
713
714 orig_flags = be16_to_cpu(ip->ip_off) & ~(IP_OFFMASK | IP_MF);
715 new_ip_off = frag_off_units | orig_flags | (more_frags ? IP_MF : 0);
716 ip->ip_off = cpu_to_be16(new_ip_off);
717 ip->ip_len = cpu_to_be16(l3hdr->iov_len + fragment_len);
718
719 eth_fix_ip4_checksum(l3hdr->iov_base, l3hdr->iov_len);
720 }
721
net_tx_pkt_do_sw_fragmentation(struct NetTxPkt * pkt,NetTxPktSend callback,void * context)722 static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
723 NetTxPktSend callback,
724 void *context)
725 {
726 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
727
728 struct iovec fragment[NET_MAX_FRAG_SG_LIST];
729 size_t fragment_len;
730 size_t l4hdr_len;
731 size_t src_len;
732
733 int src_idx, dst_idx, pl_idx;
734 size_t src_offset;
735 size_t fragment_offset = 0;
736 struct virtio_net_hdr virt_hdr = {
737 .flags = pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM ?
738 VIRTIO_NET_HDR_F_DATA_VALID : 0
739 };
740
741 /* Copy headers */
742 fragment[NET_TX_PKT_VHDR_FRAG].iov_base = &virt_hdr;
743 fragment[NET_TX_PKT_VHDR_FRAG].iov_len = sizeof(virt_hdr);
744 fragment[NET_TX_PKT_L2HDR_FRAG] = pkt->vec[NET_TX_PKT_L2HDR_FRAG];
745 fragment[NET_TX_PKT_L3HDR_FRAG] = pkt->vec[NET_TX_PKT_L3HDR_FRAG];
746
747 switch (gso_type) {
748 case VIRTIO_NET_HDR_GSO_TCPV4:
749 case VIRTIO_NET_HDR_GSO_TCPV6:
750 if (!net_tx_pkt_tcp_fragment_init(pkt, fragment, &pl_idx, &l4hdr_len,
751 &src_idx, &src_offset, &src_len)) {
752 return false;
753 }
754 break;
755
756 case VIRTIO_NET_HDR_GSO_UDP:
757 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG],
758 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1,
759 pkt->payload_len);
760 net_tx_pkt_udp_fragment_init(pkt, &pl_idx, &l4hdr_len,
761 &src_idx, &src_offset, &src_len);
762 break;
763
764 default:
765 abort();
766 }
767
768 /* Put as much data as possible and send */
769 while (true) {
770 dst_idx = pl_idx;
771 fragment_len = net_tx_pkt_fetch_fragment(pkt,
772 &src_idx, &src_offset, src_len, fragment, &dst_idx);
773 if (!fragment_len) {
774 break;
775 }
776
777 switch (gso_type) {
778 case VIRTIO_NET_HDR_GSO_TCPV4:
779 case VIRTIO_NET_HDR_GSO_TCPV6:
780 net_tx_pkt_tcp_fragment_fix(pkt, fragment, fragment_len, gso_type);
781 net_tx_pkt_do_sw_csum(pkt, fragment + NET_TX_PKT_L2HDR_FRAG,
782 dst_idx - NET_TX_PKT_L2HDR_FRAG,
783 l4hdr_len + fragment_len);
784 break;
785
786 case VIRTIO_NET_HDR_GSO_UDP:
787 net_tx_pkt_udp_fragment_fix(pkt, fragment, fragment_offset,
788 fragment_len);
789 break;
790 }
791
792 callback(context,
793 fragment + NET_TX_PKT_L2HDR_FRAG, dst_idx - NET_TX_PKT_L2HDR_FRAG,
794 fragment + NET_TX_PKT_VHDR_FRAG, dst_idx - NET_TX_PKT_VHDR_FRAG);
795
796 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
797 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
798 net_tx_pkt_tcp_fragment_advance(pkt, fragment, fragment_len,
799 gso_type);
800 }
801
802 fragment_offset += fragment_len;
803 }
804
805 if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
806 gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
807 net_tx_pkt_tcp_fragment_deinit(fragment);
808 }
809
810 return true;
811 }
812
net_tx_pkt_send(struct NetTxPkt * pkt,NetClientState * nc)813 bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc)
814 {
815 bool offload = qemu_get_vnet_hdr_len(nc->peer);
816 return net_tx_pkt_send_custom(pkt, offload, net_tx_pkt_sendv, nc);
817 }
818
net_tx_pkt_send_custom(struct NetTxPkt * pkt,bool offload,NetTxPktSend callback,void * context)819 bool net_tx_pkt_send_custom(struct NetTxPkt *pkt, bool offload,
820 NetTxPktSend callback, void *context)
821 {
822 assert(pkt);
823
824 uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
825
826 /*
827 * Since underlying infrastructure does not support IP datagrams longer
828 * than 64K we should drop such packets and don't even try to send
829 */
830 if (VIRTIO_NET_HDR_GSO_NONE != gso_type) {
831 if (pkt->payload_len >
832 ETH_MAX_IP_DGRAM_LEN -
833 pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) {
834 return false;
835 }
836 }
837
838 if (offload || gso_type == VIRTIO_NET_HDR_GSO_NONE) {
839 if (!offload && pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
840 pkt->virt_hdr.flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
841 net_tx_pkt_do_sw_csum(pkt, &pkt->vec[NET_TX_PKT_L2HDR_FRAG],
842 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1,
843 pkt->payload_len);
844 }
845
846 net_tx_pkt_fix_ip6_payload_len(pkt);
847 callback(context, pkt->vec + NET_TX_PKT_L2HDR_FRAG,
848 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_L2HDR_FRAG,
849 pkt->vec + NET_TX_PKT_VHDR_FRAG,
850 pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - NET_TX_PKT_VHDR_FRAG);
851 return true;
852 }
853
854 return net_tx_pkt_do_sw_fragmentation(pkt, callback, context);
855 }
856
net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt * pkt)857 void net_tx_pkt_fix_ip6_payload_len(struct NetTxPkt *pkt)
858 {
859 struct iovec *l2 = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
860 if (eth_get_l3_proto(l2, 1, l2->iov_len) == ETH_P_IPV6) {
861 /*
862 * TODO: if qemu would support >64K packets - add jumbo option check
863 * something like that:
864 * 'if (ip6->ip6_plen == 0 && !has_jumbo_option(ip6)) {'
865 */
866 if (pkt->l3_hdr.ip6.ip6_plen == 0) {
867 if (pkt->payload_len <= ETH_MAX_IP_DGRAM_LEN) {
868 pkt->l3_hdr.ip6.ip6_plen = htons(pkt->payload_len);
869 }
870 /*
871 * TODO: if qemu would support >64K packets
872 * add jumbo option for packets greater then 65,535 bytes
873 */
874 }
875 }
876 }
877