1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * common code for virtio vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/spinlock.h> 10 #include <linux/module.h> 11 #include <linux/sched/signal.h> 12 #include <linux/ctype.h> 13 #include <linux/list.h> 14 #include <linux/virtio.h> 15 #include <linux/virtio_ids.h> 16 #include <linux/virtio_config.h> 17 #include <linux/virtio_vsock.h> 18 #include <uapi/linux/vsockmon.h> 19 20 #include <net/sock.h> 21 #include <net/af_vsock.h> 22 23 #define CREATE_TRACE_POINTS 24 #include <trace/events/vsock_virtio_transport_common.h> 25 26 /* How long to wait for graceful shutdown of a connection */ 27 #define VSOCK_CLOSE_TIMEOUT (8 * HZ) 28 29 /* Threshold for detecting small packets to copy */ 30 #define GOOD_COPY_LEN 128 31 32 static const struct virtio_transport *virtio_transport_get_ops(void) 33 { 34 const struct vsock_transport *t = vsock_core_get_transport(); 35 36 return container_of(t, struct virtio_transport, transport); 37 } 38 39 static struct virtio_vsock_pkt * 40 virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info, 41 size_t len, 42 u32 src_cid, 43 u32 src_port, 44 u32 dst_cid, 45 u32 dst_port) 46 { 47 struct virtio_vsock_pkt *pkt; 48 int err; 49 50 pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 51 if (!pkt) 52 return NULL; 53 54 pkt->hdr.type = cpu_to_le16(info->type); 55 pkt->hdr.op = cpu_to_le16(info->op); 56 pkt->hdr.src_cid = cpu_to_le64(src_cid); 57 pkt->hdr.dst_cid = cpu_to_le64(dst_cid); 58 pkt->hdr.src_port = cpu_to_le32(src_port); 59 pkt->hdr.dst_port = cpu_to_le32(dst_port); 60 pkt->hdr.flags = cpu_to_le32(info->flags); 61 pkt->len = len; 62 pkt->hdr.len = cpu_to_le32(len); 63 pkt->reply = info->reply; 64 pkt->vsk = info->vsk; 65 66 if (info->msg && len > 0) { 67 pkt->buf = kmalloc(len, GFP_KERNEL); 68 if (!pkt->buf) 69 goto out_pkt; 70 71 pkt->buf_len = len; 72 73 err = memcpy_from_msg(pkt->buf, info->msg, len); 74 if (err) 75 goto out; 76 } 77 78 trace_virtio_transport_alloc_pkt(src_cid, src_port, 79 dst_cid, dst_port, 80 len, 81 info->type, 82 info->op, 83 info->flags); 84 85 return pkt; 86 87 out: 88 kfree(pkt->buf); 89 out_pkt: 90 kfree(pkt); 91 return NULL; 92 } 93 94 /* Packet capture */ 95 static struct sk_buff *virtio_transport_build_skb(void *opaque) 96 { 97 struct virtio_vsock_pkt *pkt = opaque; 98 struct af_vsockmon_hdr *hdr; 99 struct sk_buff *skb; 100 size_t payload_len; 101 void *payload_buf; 102 103 /* A packet could be split to fit the RX buffer, so we can retrieve 104 * the payload length from the header and the buffer pointer taking 105 * care of the offset in the original packet. 106 */ 107 payload_len = le32_to_cpu(pkt->hdr.len); 108 payload_buf = pkt->buf + pkt->off; 109 110 skb = alloc_skb(sizeof(*hdr) + sizeof(pkt->hdr) + payload_len, 111 GFP_ATOMIC); 112 if (!skb) 113 return NULL; 114 115 hdr = skb_put(skb, sizeof(*hdr)); 116 117 /* pkt->hdr is little-endian so no need to byteswap here */ 118 hdr->src_cid = pkt->hdr.src_cid; 119 hdr->src_port = pkt->hdr.src_port; 120 hdr->dst_cid = pkt->hdr.dst_cid; 121 hdr->dst_port = pkt->hdr.dst_port; 122 123 hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); 124 hdr->len = cpu_to_le16(sizeof(pkt->hdr)); 125 memset(hdr->reserved, 0, sizeof(hdr->reserved)); 126 127 switch (le16_to_cpu(pkt->hdr.op)) { 128 case VIRTIO_VSOCK_OP_REQUEST: 129 case VIRTIO_VSOCK_OP_RESPONSE: 130 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); 131 break; 132 case VIRTIO_VSOCK_OP_RST: 133 case VIRTIO_VSOCK_OP_SHUTDOWN: 134 hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT); 135 break; 136 case VIRTIO_VSOCK_OP_RW: 137 hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD); 138 break; 139 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 140 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: 141 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL); 142 break; 143 default: 144 hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN); 145 break; 146 } 147 148 skb_put_data(skb, &pkt->hdr, sizeof(pkt->hdr)); 149 150 if (payload_len) { 151 skb_put_data(skb, payload_buf, payload_len); 152 } 153 154 return skb; 155 } 156 157 void virtio_transport_deliver_tap_pkt(struct virtio_vsock_pkt *pkt) 158 { 159 vsock_deliver_tap(virtio_transport_build_skb, pkt); 160 } 161 EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); 162 163 static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, 164 struct virtio_vsock_pkt_info *info) 165 { 166 u32 src_cid, src_port, dst_cid, dst_port; 167 struct virtio_vsock_sock *vvs; 168 struct virtio_vsock_pkt *pkt; 169 u32 pkt_len = info->pkt_len; 170 171 src_cid = vm_sockets_get_local_cid(); 172 src_port = vsk->local_addr.svm_port; 173 if (!info->remote_cid) { 174 dst_cid = vsk->remote_addr.svm_cid; 175 dst_port = vsk->remote_addr.svm_port; 176 } else { 177 dst_cid = info->remote_cid; 178 dst_port = info->remote_port; 179 } 180 181 vvs = vsk->trans; 182 183 /* we can send less than pkt_len bytes */ 184 if (pkt_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) 185 pkt_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; 186 187 /* virtio_transport_get_credit might return less than pkt_len credit */ 188 pkt_len = virtio_transport_get_credit(vvs, pkt_len); 189 190 /* Do not send zero length OP_RW pkt */ 191 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 192 return pkt_len; 193 194 pkt = virtio_transport_alloc_pkt(info, pkt_len, 195 src_cid, src_port, 196 dst_cid, dst_port); 197 if (!pkt) { 198 virtio_transport_put_credit(vvs, pkt_len); 199 return -ENOMEM; 200 } 201 202 virtio_transport_inc_tx_pkt(vvs, pkt); 203 204 return virtio_transport_get_ops()->send_pkt(pkt); 205 } 206 207 static void virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 208 struct virtio_vsock_pkt *pkt) 209 { 210 vvs->rx_bytes += pkt->len; 211 } 212 213 static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 214 struct virtio_vsock_pkt *pkt) 215 { 216 vvs->rx_bytes -= pkt->len; 217 vvs->fwd_cnt += pkt->len; 218 } 219 220 void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) 221 { 222 spin_lock_bh(&vvs->rx_lock); 223 vvs->last_fwd_cnt = vvs->fwd_cnt; 224 pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 225 pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); 226 spin_unlock_bh(&vvs->rx_lock); 227 } 228 EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); 229 230 u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) 231 { 232 u32 ret; 233 234 spin_lock_bh(&vvs->tx_lock); 235 ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 236 if (ret > credit) 237 ret = credit; 238 vvs->tx_cnt += ret; 239 spin_unlock_bh(&vvs->tx_lock); 240 241 return ret; 242 } 243 EXPORT_SYMBOL_GPL(virtio_transport_get_credit); 244 245 void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) 246 { 247 spin_lock_bh(&vvs->tx_lock); 248 vvs->tx_cnt -= credit; 249 spin_unlock_bh(&vvs->tx_lock); 250 } 251 EXPORT_SYMBOL_GPL(virtio_transport_put_credit); 252 253 static int virtio_transport_send_credit_update(struct vsock_sock *vsk, 254 int type, 255 struct virtio_vsock_hdr *hdr) 256 { 257 struct virtio_vsock_pkt_info info = { 258 .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, 259 .type = type, 260 .vsk = vsk, 261 }; 262 263 return virtio_transport_send_pkt_info(vsk, &info); 264 } 265 266 static ssize_t 267 virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, 268 struct msghdr *msg, 269 size_t len) 270 { 271 struct virtio_vsock_sock *vvs = vsk->trans; 272 struct virtio_vsock_pkt *pkt; 273 size_t bytes, total = 0; 274 u32 free_space; 275 int err = -EFAULT; 276 277 spin_lock_bh(&vvs->rx_lock); 278 while (total < len && !list_empty(&vvs->rx_queue)) { 279 pkt = list_first_entry(&vvs->rx_queue, 280 struct virtio_vsock_pkt, list); 281 282 bytes = len - total; 283 if (bytes > pkt->len - pkt->off) 284 bytes = pkt->len - pkt->off; 285 286 /* sk_lock is held by caller so no one else can dequeue. 287 * Unlock rx_lock since memcpy_to_msg() may sleep. 288 */ 289 spin_unlock_bh(&vvs->rx_lock); 290 291 err = memcpy_to_msg(msg, pkt->buf + pkt->off, bytes); 292 if (err) 293 goto out; 294 295 spin_lock_bh(&vvs->rx_lock); 296 297 total += bytes; 298 pkt->off += bytes; 299 if (pkt->off == pkt->len) { 300 virtio_transport_dec_rx_pkt(vvs, pkt); 301 list_del(&pkt->list); 302 virtio_transport_free_pkt(pkt); 303 } 304 } 305 306 free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); 307 308 spin_unlock_bh(&vvs->rx_lock); 309 310 /* We send a credit update only when the space available seen 311 * by the transmitter is less than VIRTIO_VSOCK_MAX_PKT_BUF_SIZE 312 */ 313 if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { 314 virtio_transport_send_credit_update(vsk, 315 VIRTIO_VSOCK_TYPE_STREAM, 316 NULL); 317 } 318 319 return total; 320 321 out: 322 if (total) 323 err = total; 324 return err; 325 } 326 327 ssize_t 328 virtio_transport_stream_dequeue(struct vsock_sock *vsk, 329 struct msghdr *msg, 330 size_t len, int flags) 331 { 332 if (flags & MSG_PEEK) 333 return -EOPNOTSUPP; 334 335 return virtio_transport_stream_do_dequeue(vsk, msg, len); 336 } 337 EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); 338 339 int 340 virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 341 struct msghdr *msg, 342 size_t len, int flags) 343 { 344 return -EOPNOTSUPP; 345 } 346 EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); 347 348 s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) 349 { 350 struct virtio_vsock_sock *vvs = vsk->trans; 351 s64 bytes; 352 353 spin_lock_bh(&vvs->rx_lock); 354 bytes = vvs->rx_bytes; 355 spin_unlock_bh(&vvs->rx_lock); 356 357 return bytes; 358 } 359 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); 360 361 static s64 virtio_transport_has_space(struct vsock_sock *vsk) 362 { 363 struct virtio_vsock_sock *vvs = vsk->trans; 364 s64 bytes; 365 366 bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 367 if (bytes < 0) 368 bytes = 0; 369 370 return bytes; 371 } 372 373 s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) 374 { 375 struct virtio_vsock_sock *vvs = vsk->trans; 376 s64 bytes; 377 378 spin_lock_bh(&vvs->tx_lock); 379 bytes = virtio_transport_has_space(vsk); 380 spin_unlock_bh(&vvs->tx_lock); 381 382 return bytes; 383 } 384 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); 385 386 int virtio_transport_do_socket_init(struct vsock_sock *vsk, 387 struct vsock_sock *psk) 388 { 389 struct virtio_vsock_sock *vvs; 390 391 vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); 392 if (!vvs) 393 return -ENOMEM; 394 395 vsk->trans = vvs; 396 vvs->vsk = vsk; 397 if (psk) { 398 struct virtio_vsock_sock *ptrans = psk->trans; 399 400 vvs->buf_size = ptrans->buf_size; 401 vvs->buf_size_min = ptrans->buf_size_min; 402 vvs->buf_size_max = ptrans->buf_size_max; 403 vvs->peer_buf_alloc = ptrans->peer_buf_alloc; 404 } else { 405 vvs->buf_size = VIRTIO_VSOCK_DEFAULT_BUF_SIZE; 406 vvs->buf_size_min = VIRTIO_VSOCK_DEFAULT_MIN_BUF_SIZE; 407 vvs->buf_size_max = VIRTIO_VSOCK_DEFAULT_MAX_BUF_SIZE; 408 } 409 410 vvs->buf_alloc = vvs->buf_size; 411 412 spin_lock_init(&vvs->rx_lock); 413 spin_lock_init(&vvs->tx_lock); 414 INIT_LIST_HEAD(&vvs->rx_queue); 415 416 return 0; 417 } 418 EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); 419 420 u64 virtio_transport_get_buffer_size(struct vsock_sock *vsk) 421 { 422 struct virtio_vsock_sock *vvs = vsk->trans; 423 424 return vvs->buf_size; 425 } 426 EXPORT_SYMBOL_GPL(virtio_transport_get_buffer_size); 427 428 u64 virtio_transport_get_min_buffer_size(struct vsock_sock *vsk) 429 { 430 struct virtio_vsock_sock *vvs = vsk->trans; 431 432 return vvs->buf_size_min; 433 } 434 EXPORT_SYMBOL_GPL(virtio_transport_get_min_buffer_size); 435 436 u64 virtio_transport_get_max_buffer_size(struct vsock_sock *vsk) 437 { 438 struct virtio_vsock_sock *vvs = vsk->trans; 439 440 return vvs->buf_size_max; 441 } 442 EXPORT_SYMBOL_GPL(virtio_transport_get_max_buffer_size); 443 444 void virtio_transport_set_buffer_size(struct vsock_sock *vsk, u64 val) 445 { 446 struct virtio_vsock_sock *vvs = vsk->trans; 447 448 if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 449 val = VIRTIO_VSOCK_MAX_BUF_SIZE; 450 if (val < vvs->buf_size_min) 451 vvs->buf_size_min = val; 452 if (val > vvs->buf_size_max) 453 vvs->buf_size_max = val; 454 vvs->buf_size = val; 455 vvs->buf_alloc = val; 456 } 457 EXPORT_SYMBOL_GPL(virtio_transport_set_buffer_size); 458 459 void virtio_transport_set_min_buffer_size(struct vsock_sock *vsk, u64 val) 460 { 461 struct virtio_vsock_sock *vvs = vsk->trans; 462 463 if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 464 val = VIRTIO_VSOCK_MAX_BUF_SIZE; 465 if (val > vvs->buf_size) 466 vvs->buf_size = val; 467 vvs->buf_size_min = val; 468 } 469 EXPORT_SYMBOL_GPL(virtio_transport_set_min_buffer_size); 470 471 void virtio_transport_set_max_buffer_size(struct vsock_sock *vsk, u64 val) 472 { 473 struct virtio_vsock_sock *vvs = vsk->trans; 474 475 if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 476 val = VIRTIO_VSOCK_MAX_BUF_SIZE; 477 if (val < vvs->buf_size) 478 vvs->buf_size = val; 479 vvs->buf_size_max = val; 480 } 481 EXPORT_SYMBOL_GPL(virtio_transport_set_max_buffer_size); 482 483 int 484 virtio_transport_notify_poll_in(struct vsock_sock *vsk, 485 size_t target, 486 bool *data_ready_now) 487 { 488 if (vsock_stream_has_data(vsk)) 489 *data_ready_now = true; 490 else 491 *data_ready_now = false; 492 493 return 0; 494 } 495 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); 496 497 int 498 virtio_transport_notify_poll_out(struct vsock_sock *vsk, 499 size_t target, 500 bool *space_avail_now) 501 { 502 s64 free_space; 503 504 free_space = vsock_stream_has_space(vsk); 505 if (free_space > 0) 506 *space_avail_now = true; 507 else if (free_space == 0) 508 *space_avail_now = false; 509 510 return 0; 511 } 512 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); 513 514 int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 515 size_t target, struct vsock_transport_recv_notify_data *data) 516 { 517 return 0; 518 } 519 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); 520 521 int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 522 size_t target, struct vsock_transport_recv_notify_data *data) 523 { 524 return 0; 525 } 526 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); 527 528 int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 529 size_t target, struct vsock_transport_recv_notify_data *data) 530 { 531 return 0; 532 } 533 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); 534 535 int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 536 size_t target, ssize_t copied, bool data_read, 537 struct vsock_transport_recv_notify_data *data) 538 { 539 return 0; 540 } 541 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); 542 543 int virtio_transport_notify_send_init(struct vsock_sock *vsk, 544 struct vsock_transport_send_notify_data *data) 545 { 546 return 0; 547 } 548 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); 549 550 int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 551 struct vsock_transport_send_notify_data *data) 552 { 553 return 0; 554 } 555 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); 556 557 int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 558 struct vsock_transport_send_notify_data *data) 559 { 560 return 0; 561 } 562 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); 563 564 int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 565 ssize_t written, struct vsock_transport_send_notify_data *data) 566 { 567 return 0; 568 } 569 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); 570 571 u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) 572 { 573 struct virtio_vsock_sock *vvs = vsk->trans; 574 575 return vvs->buf_size; 576 } 577 EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); 578 579 bool virtio_transport_stream_is_active(struct vsock_sock *vsk) 580 { 581 return true; 582 } 583 EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); 584 585 bool virtio_transport_stream_allow(u32 cid, u32 port) 586 { 587 return true; 588 } 589 EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); 590 591 int virtio_transport_dgram_bind(struct vsock_sock *vsk, 592 struct sockaddr_vm *addr) 593 { 594 return -EOPNOTSUPP; 595 } 596 EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); 597 598 bool virtio_transport_dgram_allow(u32 cid, u32 port) 599 { 600 return false; 601 } 602 EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); 603 604 int virtio_transport_connect(struct vsock_sock *vsk) 605 { 606 struct virtio_vsock_pkt_info info = { 607 .op = VIRTIO_VSOCK_OP_REQUEST, 608 .type = VIRTIO_VSOCK_TYPE_STREAM, 609 .vsk = vsk, 610 }; 611 612 return virtio_transport_send_pkt_info(vsk, &info); 613 } 614 EXPORT_SYMBOL_GPL(virtio_transport_connect); 615 616 int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) 617 { 618 struct virtio_vsock_pkt_info info = { 619 .op = VIRTIO_VSOCK_OP_SHUTDOWN, 620 .type = VIRTIO_VSOCK_TYPE_STREAM, 621 .flags = (mode & RCV_SHUTDOWN ? 622 VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | 623 (mode & SEND_SHUTDOWN ? 624 VIRTIO_VSOCK_SHUTDOWN_SEND : 0), 625 .vsk = vsk, 626 }; 627 628 return virtio_transport_send_pkt_info(vsk, &info); 629 } 630 EXPORT_SYMBOL_GPL(virtio_transport_shutdown); 631 632 int 633 virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 634 struct sockaddr_vm *remote_addr, 635 struct msghdr *msg, 636 size_t dgram_len) 637 { 638 return -EOPNOTSUPP; 639 } 640 EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); 641 642 ssize_t 643 virtio_transport_stream_enqueue(struct vsock_sock *vsk, 644 struct msghdr *msg, 645 size_t len) 646 { 647 struct virtio_vsock_pkt_info info = { 648 .op = VIRTIO_VSOCK_OP_RW, 649 .type = VIRTIO_VSOCK_TYPE_STREAM, 650 .msg = msg, 651 .pkt_len = len, 652 .vsk = vsk, 653 }; 654 655 return virtio_transport_send_pkt_info(vsk, &info); 656 } 657 EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); 658 659 void virtio_transport_destruct(struct vsock_sock *vsk) 660 { 661 struct virtio_vsock_sock *vvs = vsk->trans; 662 663 kfree(vvs); 664 } 665 EXPORT_SYMBOL_GPL(virtio_transport_destruct); 666 667 static int virtio_transport_reset(struct vsock_sock *vsk, 668 struct virtio_vsock_pkt *pkt) 669 { 670 struct virtio_vsock_pkt_info info = { 671 .op = VIRTIO_VSOCK_OP_RST, 672 .type = VIRTIO_VSOCK_TYPE_STREAM, 673 .reply = !!pkt, 674 .vsk = vsk, 675 }; 676 677 /* Send RST only if the original pkt is not a RST pkt */ 678 if (pkt && le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 679 return 0; 680 681 return virtio_transport_send_pkt_info(vsk, &info); 682 } 683 684 /* Normally packets are associated with a socket. There may be no socket if an 685 * attempt was made to connect to a socket that does not exist. 686 */ 687 static int virtio_transport_reset_no_sock(struct virtio_vsock_pkt *pkt) 688 { 689 const struct virtio_transport *t; 690 struct virtio_vsock_pkt *reply; 691 struct virtio_vsock_pkt_info info = { 692 .op = VIRTIO_VSOCK_OP_RST, 693 .type = le16_to_cpu(pkt->hdr.type), 694 .reply = true, 695 }; 696 697 /* Send RST only if the original pkt is not a RST pkt */ 698 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 699 return 0; 700 701 reply = virtio_transport_alloc_pkt(&info, 0, 702 le64_to_cpu(pkt->hdr.dst_cid), 703 le32_to_cpu(pkt->hdr.dst_port), 704 le64_to_cpu(pkt->hdr.src_cid), 705 le32_to_cpu(pkt->hdr.src_port)); 706 if (!reply) 707 return -ENOMEM; 708 709 t = virtio_transport_get_ops(); 710 if (!t) { 711 virtio_transport_free_pkt(reply); 712 return -ENOTCONN; 713 } 714 715 return t->send_pkt(reply); 716 } 717 718 static void virtio_transport_wait_close(struct sock *sk, long timeout) 719 { 720 if (timeout) { 721 DEFINE_WAIT_FUNC(wait, woken_wake_function); 722 723 add_wait_queue(sk_sleep(sk), &wait); 724 725 do { 726 if (sk_wait_event(sk, &timeout, 727 sock_flag(sk, SOCK_DONE), &wait)) 728 break; 729 } while (!signal_pending(current) && timeout); 730 731 remove_wait_queue(sk_sleep(sk), &wait); 732 } 733 } 734 735 static void virtio_transport_do_close(struct vsock_sock *vsk, 736 bool cancel_timeout) 737 { 738 struct sock *sk = sk_vsock(vsk); 739 740 sock_set_flag(sk, SOCK_DONE); 741 vsk->peer_shutdown = SHUTDOWN_MASK; 742 if (vsock_stream_has_data(vsk) <= 0) 743 sk->sk_state = TCP_CLOSING; 744 sk->sk_state_change(sk); 745 746 if (vsk->close_work_scheduled && 747 (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { 748 vsk->close_work_scheduled = false; 749 750 vsock_remove_sock(vsk); 751 752 /* Release refcnt obtained when we scheduled the timeout */ 753 sock_put(sk); 754 } 755 } 756 757 static void virtio_transport_close_timeout(struct work_struct *work) 758 { 759 struct vsock_sock *vsk = 760 container_of(work, struct vsock_sock, close_work.work); 761 struct sock *sk = sk_vsock(vsk); 762 763 sock_hold(sk); 764 lock_sock(sk); 765 766 if (!sock_flag(sk, SOCK_DONE)) { 767 (void)virtio_transport_reset(vsk, NULL); 768 769 virtio_transport_do_close(vsk, false); 770 } 771 772 vsk->close_work_scheduled = false; 773 774 release_sock(sk); 775 sock_put(sk); 776 } 777 778 /* User context, vsk->sk is locked */ 779 static bool virtio_transport_close(struct vsock_sock *vsk) 780 { 781 struct sock *sk = &vsk->sk; 782 783 if (!(sk->sk_state == TCP_ESTABLISHED || 784 sk->sk_state == TCP_CLOSING)) 785 return true; 786 787 /* Already received SHUTDOWN from peer, reply with RST */ 788 if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { 789 (void)virtio_transport_reset(vsk, NULL); 790 return true; 791 } 792 793 if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) 794 (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); 795 796 if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) 797 virtio_transport_wait_close(sk, sk->sk_lingertime); 798 799 if (sock_flag(sk, SOCK_DONE)) { 800 return true; 801 } 802 803 sock_hold(sk); 804 INIT_DELAYED_WORK(&vsk->close_work, 805 virtio_transport_close_timeout); 806 vsk->close_work_scheduled = true; 807 schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); 808 return false; 809 } 810 811 void virtio_transport_release(struct vsock_sock *vsk) 812 { 813 struct virtio_vsock_sock *vvs = vsk->trans; 814 struct virtio_vsock_pkt *pkt, *tmp; 815 struct sock *sk = &vsk->sk; 816 bool remove_sock = true; 817 818 lock_sock(sk); 819 if (sk->sk_type == SOCK_STREAM) 820 remove_sock = virtio_transport_close(vsk); 821 822 list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { 823 list_del(&pkt->list); 824 virtio_transport_free_pkt(pkt); 825 } 826 release_sock(sk); 827 828 if (remove_sock) 829 vsock_remove_sock(vsk); 830 } 831 EXPORT_SYMBOL_GPL(virtio_transport_release); 832 833 static int 834 virtio_transport_recv_connecting(struct sock *sk, 835 struct virtio_vsock_pkt *pkt) 836 { 837 struct vsock_sock *vsk = vsock_sk(sk); 838 int err; 839 int skerr; 840 841 switch (le16_to_cpu(pkt->hdr.op)) { 842 case VIRTIO_VSOCK_OP_RESPONSE: 843 sk->sk_state = TCP_ESTABLISHED; 844 sk->sk_socket->state = SS_CONNECTED; 845 vsock_insert_connected(vsk); 846 sk->sk_state_change(sk); 847 break; 848 case VIRTIO_VSOCK_OP_INVALID: 849 break; 850 case VIRTIO_VSOCK_OP_RST: 851 skerr = ECONNRESET; 852 err = 0; 853 goto destroy; 854 default: 855 skerr = EPROTO; 856 err = -EINVAL; 857 goto destroy; 858 } 859 return 0; 860 861 destroy: 862 virtio_transport_reset(vsk, pkt); 863 sk->sk_state = TCP_CLOSE; 864 sk->sk_err = skerr; 865 sk->sk_error_report(sk); 866 return err; 867 } 868 869 static void 870 virtio_transport_recv_enqueue(struct vsock_sock *vsk, 871 struct virtio_vsock_pkt *pkt) 872 { 873 struct virtio_vsock_sock *vvs = vsk->trans; 874 bool free_pkt = false; 875 876 pkt->len = le32_to_cpu(pkt->hdr.len); 877 pkt->off = 0; 878 879 spin_lock_bh(&vvs->rx_lock); 880 881 virtio_transport_inc_rx_pkt(vvs, pkt); 882 883 /* Try to copy small packets into the buffer of last packet queued, 884 * to avoid wasting memory queueing the entire buffer with a small 885 * payload. 886 */ 887 if (pkt->len <= GOOD_COPY_LEN && !list_empty(&vvs->rx_queue)) { 888 struct virtio_vsock_pkt *last_pkt; 889 890 last_pkt = list_last_entry(&vvs->rx_queue, 891 struct virtio_vsock_pkt, list); 892 893 /* If there is space in the last packet queued, we copy the 894 * new packet in its buffer. 895 */ 896 if (pkt->len <= last_pkt->buf_len - last_pkt->len) { 897 memcpy(last_pkt->buf + last_pkt->len, pkt->buf, 898 pkt->len); 899 last_pkt->len += pkt->len; 900 free_pkt = true; 901 goto out; 902 } 903 } 904 905 list_add_tail(&pkt->list, &vvs->rx_queue); 906 907 out: 908 spin_unlock_bh(&vvs->rx_lock); 909 if (free_pkt) 910 virtio_transport_free_pkt(pkt); 911 } 912 913 static int 914 virtio_transport_recv_connected(struct sock *sk, 915 struct virtio_vsock_pkt *pkt) 916 { 917 struct vsock_sock *vsk = vsock_sk(sk); 918 int err = 0; 919 920 switch (le16_to_cpu(pkt->hdr.op)) { 921 case VIRTIO_VSOCK_OP_RW: 922 virtio_transport_recv_enqueue(vsk, pkt); 923 sk->sk_data_ready(sk); 924 return err; 925 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 926 sk->sk_write_space(sk); 927 break; 928 case VIRTIO_VSOCK_OP_SHUTDOWN: 929 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 930 vsk->peer_shutdown |= RCV_SHUTDOWN; 931 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 932 vsk->peer_shutdown |= SEND_SHUTDOWN; 933 if (vsk->peer_shutdown == SHUTDOWN_MASK && 934 vsock_stream_has_data(vsk) <= 0) { 935 sock_set_flag(sk, SOCK_DONE); 936 sk->sk_state = TCP_CLOSING; 937 } 938 if (le32_to_cpu(pkt->hdr.flags)) 939 sk->sk_state_change(sk); 940 break; 941 case VIRTIO_VSOCK_OP_RST: 942 virtio_transport_do_close(vsk, true); 943 break; 944 default: 945 err = -EINVAL; 946 break; 947 } 948 949 virtio_transport_free_pkt(pkt); 950 return err; 951 } 952 953 static void 954 virtio_transport_recv_disconnecting(struct sock *sk, 955 struct virtio_vsock_pkt *pkt) 956 { 957 struct vsock_sock *vsk = vsock_sk(sk); 958 959 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 960 virtio_transport_do_close(vsk, true); 961 } 962 963 static int 964 virtio_transport_send_response(struct vsock_sock *vsk, 965 struct virtio_vsock_pkt *pkt) 966 { 967 struct virtio_vsock_pkt_info info = { 968 .op = VIRTIO_VSOCK_OP_RESPONSE, 969 .type = VIRTIO_VSOCK_TYPE_STREAM, 970 .remote_cid = le64_to_cpu(pkt->hdr.src_cid), 971 .remote_port = le32_to_cpu(pkt->hdr.src_port), 972 .reply = true, 973 .vsk = vsk, 974 }; 975 976 return virtio_transport_send_pkt_info(vsk, &info); 977 } 978 979 /* Handle server socket */ 980 static int 981 virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt) 982 { 983 struct vsock_sock *vsk = vsock_sk(sk); 984 struct vsock_sock *vchild; 985 struct sock *child; 986 987 if (le16_to_cpu(pkt->hdr.op) != VIRTIO_VSOCK_OP_REQUEST) { 988 virtio_transport_reset(vsk, pkt); 989 return -EINVAL; 990 } 991 992 if (sk_acceptq_is_full(sk)) { 993 virtio_transport_reset(vsk, pkt); 994 return -ENOMEM; 995 } 996 997 child = __vsock_create(sock_net(sk), NULL, sk, GFP_KERNEL, 998 sk->sk_type, 0); 999 if (!child) { 1000 virtio_transport_reset(vsk, pkt); 1001 return -ENOMEM; 1002 } 1003 1004 sk->sk_ack_backlog++; 1005 1006 lock_sock_nested(child, SINGLE_DEPTH_NESTING); 1007 1008 child->sk_state = TCP_ESTABLISHED; 1009 1010 vchild = vsock_sk(child); 1011 vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid), 1012 le32_to_cpu(pkt->hdr.dst_port)); 1013 vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid), 1014 le32_to_cpu(pkt->hdr.src_port)); 1015 1016 vsock_insert_connected(vchild); 1017 vsock_enqueue_accept(sk, child); 1018 virtio_transport_send_response(vchild, pkt); 1019 1020 release_sock(child); 1021 1022 sk->sk_data_ready(sk); 1023 return 0; 1024 } 1025 1026 static bool virtio_transport_space_update(struct sock *sk, 1027 struct virtio_vsock_pkt *pkt) 1028 { 1029 struct vsock_sock *vsk = vsock_sk(sk); 1030 struct virtio_vsock_sock *vvs = vsk->trans; 1031 bool space_available; 1032 1033 /* buf_alloc and fwd_cnt is always included in the hdr */ 1034 spin_lock_bh(&vvs->tx_lock); 1035 vvs->peer_buf_alloc = le32_to_cpu(pkt->hdr.buf_alloc); 1036 vvs->peer_fwd_cnt = le32_to_cpu(pkt->hdr.fwd_cnt); 1037 space_available = virtio_transport_has_space(vsk); 1038 spin_unlock_bh(&vvs->tx_lock); 1039 return space_available; 1040 } 1041 1042 /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex 1043 * lock. 1044 */ 1045 void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt) 1046 { 1047 struct sockaddr_vm src, dst; 1048 struct vsock_sock *vsk; 1049 struct sock *sk; 1050 bool space_available; 1051 1052 vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid), 1053 le32_to_cpu(pkt->hdr.src_port)); 1054 vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid), 1055 le32_to_cpu(pkt->hdr.dst_port)); 1056 1057 trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 1058 dst.svm_cid, dst.svm_port, 1059 le32_to_cpu(pkt->hdr.len), 1060 le16_to_cpu(pkt->hdr.type), 1061 le16_to_cpu(pkt->hdr.op), 1062 le32_to_cpu(pkt->hdr.flags), 1063 le32_to_cpu(pkt->hdr.buf_alloc), 1064 le32_to_cpu(pkt->hdr.fwd_cnt)); 1065 1066 if (le16_to_cpu(pkt->hdr.type) != VIRTIO_VSOCK_TYPE_STREAM) { 1067 (void)virtio_transport_reset_no_sock(pkt); 1068 goto free_pkt; 1069 } 1070 1071 /* The socket must be in connected or bound table 1072 * otherwise send reset back 1073 */ 1074 sk = vsock_find_connected_socket(&src, &dst); 1075 if (!sk) { 1076 sk = vsock_find_bound_socket(&dst); 1077 if (!sk) { 1078 (void)virtio_transport_reset_no_sock(pkt); 1079 goto free_pkt; 1080 } 1081 } 1082 1083 vsk = vsock_sk(sk); 1084 1085 space_available = virtio_transport_space_update(sk, pkt); 1086 1087 lock_sock(sk); 1088 1089 /* Update CID in case it has changed after a transport reset event */ 1090 vsk->local_addr.svm_cid = dst.svm_cid; 1091 1092 if (space_available) 1093 sk->sk_write_space(sk); 1094 1095 switch (sk->sk_state) { 1096 case TCP_LISTEN: 1097 virtio_transport_recv_listen(sk, pkt); 1098 virtio_transport_free_pkt(pkt); 1099 break; 1100 case TCP_SYN_SENT: 1101 virtio_transport_recv_connecting(sk, pkt); 1102 virtio_transport_free_pkt(pkt); 1103 break; 1104 case TCP_ESTABLISHED: 1105 virtio_transport_recv_connected(sk, pkt); 1106 break; 1107 case TCP_CLOSING: 1108 virtio_transport_recv_disconnecting(sk, pkt); 1109 virtio_transport_free_pkt(pkt); 1110 break; 1111 default: 1112 virtio_transport_free_pkt(pkt); 1113 break; 1114 } 1115 release_sock(sk); 1116 1117 /* Release refcnt obtained when we fetched this socket out of the 1118 * bound or connected list. 1119 */ 1120 sock_put(sk); 1121 return; 1122 1123 free_pkt: 1124 virtio_transport_free_pkt(pkt); 1125 } 1126 EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 1127 1128 void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) 1129 { 1130 kfree(pkt->buf); 1131 kfree(pkt); 1132 } 1133 EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); 1134 1135 MODULE_LICENSE("GPL v2"); 1136 MODULE_AUTHOR("Asias He"); 1137 MODULE_DESCRIPTION("common code for virtio vsock"); 1138