1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * common code for virtio vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/spinlock.h> 10 #include <linux/module.h> 11 #include <linux/sched/signal.h> 12 #include <linux/ctype.h> 13 #include <linux/list.h> 14 #include <linux/virtio_vsock.h> 15 #include <uapi/linux/vsockmon.h> 16 17 #include <net/sock.h> 18 #include <net/af_vsock.h> 19 20 #define CREATE_TRACE_POINTS 21 #include <trace/events/vsock_virtio_transport_common.h> 22 23 /* How long to wait for graceful shutdown of a connection */ 24 #define VSOCK_CLOSE_TIMEOUT (8 * HZ) 25 26 /* Threshold for detecting small packets to copy */ 27 #define GOOD_COPY_LEN 128 28 29 static const struct virtio_transport * 30 virtio_transport_get_ops(struct vsock_sock *vsk) 31 { 32 const struct vsock_transport *t = vsock_core_get_transport(vsk); 33 34 if (WARN_ON(!t)) 35 return NULL; 36 37 return container_of(t, struct virtio_transport, transport); 38 } 39 40 /* Returns a new packet on success, otherwise returns NULL. 41 * 42 * If NULL is returned, errp is set to a negative errno. 43 */ 44 static struct sk_buff * 45 virtio_transport_alloc_skb(struct virtio_vsock_pkt_info *info, 46 size_t len, 47 u32 src_cid, 48 u32 src_port, 49 u32 dst_cid, 50 u32 dst_port) 51 { 52 const size_t skb_len = VIRTIO_VSOCK_SKB_HEADROOM + len; 53 struct virtio_vsock_hdr *hdr; 54 struct sk_buff *skb; 55 void *payload; 56 int err; 57 58 skb = virtio_vsock_alloc_skb(skb_len, GFP_KERNEL); 59 if (!skb) 60 return NULL; 61 62 hdr = virtio_vsock_hdr(skb); 63 hdr->type = cpu_to_le16(info->type); 64 hdr->op = cpu_to_le16(info->op); 65 hdr->src_cid = cpu_to_le64(src_cid); 66 hdr->dst_cid = cpu_to_le64(dst_cid); 67 hdr->src_port = cpu_to_le32(src_port); 68 hdr->dst_port = cpu_to_le32(dst_port); 69 hdr->flags = cpu_to_le32(info->flags); 70 hdr->len = cpu_to_le32(len); 71 hdr->buf_alloc = cpu_to_le32(0); 72 hdr->fwd_cnt = cpu_to_le32(0); 73 74 if (info->msg && len > 0) { 75 payload = skb_put(skb, len); 76 err = memcpy_from_msg(payload, info->msg, len); 77 if (err) 78 goto out; 79 80 if (msg_data_left(info->msg) == 0 && 81 info->type == VIRTIO_VSOCK_TYPE_SEQPACKET) { 82 hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOM); 83 84 if (info->msg->msg_flags & MSG_EOR) 85 hdr->flags |= cpu_to_le32(VIRTIO_VSOCK_SEQ_EOR); 86 } 87 } 88 89 if (info->reply) 90 virtio_vsock_skb_set_reply(skb); 91 92 trace_virtio_transport_alloc_pkt(src_cid, src_port, 93 dst_cid, dst_port, 94 len, 95 info->type, 96 info->op, 97 info->flags); 98 99 if (info->vsk && !skb_set_owner_sk_safe(skb, sk_vsock(info->vsk))) { 100 WARN_ONCE(1, "failed to allocate skb on vsock socket with sk_refcnt == 0\n"); 101 goto out; 102 } 103 104 return skb; 105 106 out: 107 kfree_skb(skb); 108 return NULL; 109 } 110 111 /* Packet capture */ 112 static struct sk_buff *virtio_transport_build_skb(void *opaque) 113 { 114 struct virtio_vsock_hdr *pkt_hdr; 115 struct sk_buff *pkt = opaque; 116 struct af_vsockmon_hdr *hdr; 117 struct sk_buff *skb; 118 size_t payload_len; 119 void *payload_buf; 120 121 /* A packet could be split to fit the RX buffer, so we can retrieve 122 * the payload length from the header and the buffer pointer taking 123 * care of the offset in the original packet. 124 */ 125 pkt_hdr = virtio_vsock_hdr(pkt); 126 payload_len = pkt->len; 127 payload_buf = pkt->data; 128 129 skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len, 130 GFP_ATOMIC); 131 if (!skb) 132 return NULL; 133 134 hdr = skb_put(skb, sizeof(*hdr)); 135 136 /* pkt->hdr is little-endian so no need to byteswap here */ 137 hdr->src_cid = pkt_hdr->src_cid; 138 hdr->src_port = pkt_hdr->src_port; 139 hdr->dst_cid = pkt_hdr->dst_cid; 140 hdr->dst_port = pkt_hdr->dst_port; 141 142 hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); 143 hdr->len = cpu_to_le16(sizeof(*pkt_hdr)); 144 memset(hdr->reserved, 0, sizeof(hdr->reserved)); 145 146 switch (le16_to_cpu(pkt_hdr->op)) { 147 case VIRTIO_VSOCK_OP_REQUEST: 148 case VIRTIO_VSOCK_OP_RESPONSE: 149 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); 150 break; 151 case VIRTIO_VSOCK_OP_RST: 152 case VIRTIO_VSOCK_OP_SHUTDOWN: 153 hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT); 154 break; 155 case VIRTIO_VSOCK_OP_RW: 156 hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD); 157 break; 158 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 159 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: 160 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL); 161 break; 162 default: 163 hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN); 164 break; 165 } 166 167 skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr)); 168 169 if (payload_len) { 170 skb_put_data(skb, payload_buf, payload_len); 171 } 172 173 return skb; 174 } 175 176 void virtio_transport_deliver_tap_pkt(struct sk_buff *skb) 177 { 178 if (virtio_vsock_skb_tap_delivered(skb)) 179 return; 180 181 vsock_deliver_tap(virtio_transport_build_skb, skb); 182 virtio_vsock_skb_set_tap_delivered(skb); 183 } 184 EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); 185 186 static u16 virtio_transport_get_type(struct sock *sk) 187 { 188 if (sk->sk_type == SOCK_STREAM) 189 return VIRTIO_VSOCK_TYPE_STREAM; 190 else 191 return VIRTIO_VSOCK_TYPE_SEQPACKET; 192 } 193 194 /* This function can only be used on connecting/connected sockets, 195 * since a socket assigned to a transport is required. 196 * 197 * Do not use on listener sockets! 198 */ 199 static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, 200 struct virtio_vsock_pkt_info *info) 201 { 202 u32 src_cid, src_port, dst_cid, dst_port; 203 const struct virtio_transport *t_ops; 204 struct virtio_vsock_sock *vvs; 205 u32 pkt_len = info->pkt_len; 206 u32 rest_len; 207 int ret; 208 209 info->type = virtio_transport_get_type(sk_vsock(vsk)); 210 211 t_ops = virtio_transport_get_ops(vsk); 212 if (unlikely(!t_ops)) 213 return -EFAULT; 214 215 src_cid = t_ops->transport.get_local_cid(); 216 src_port = vsk->local_addr.svm_port; 217 if (!info->remote_cid) { 218 dst_cid = vsk->remote_addr.svm_cid; 219 dst_port = vsk->remote_addr.svm_port; 220 } else { 221 dst_cid = info->remote_cid; 222 dst_port = info->remote_port; 223 } 224 225 vvs = vsk->trans; 226 227 /* virtio_transport_get_credit might return less than pkt_len credit */ 228 pkt_len = virtio_transport_get_credit(vvs, pkt_len); 229 230 /* Do not send zero length OP_RW pkt */ 231 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 232 return pkt_len; 233 234 rest_len = pkt_len; 235 236 do { 237 struct sk_buff *skb; 238 size_t skb_len; 239 240 skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE, rest_len); 241 242 skb = virtio_transport_alloc_skb(info, skb_len, 243 src_cid, src_port, 244 dst_cid, dst_port); 245 if (!skb) { 246 ret = -ENOMEM; 247 break; 248 } 249 250 virtio_transport_inc_tx_pkt(vvs, skb); 251 252 ret = t_ops->send_pkt(skb); 253 if (ret < 0) 254 break; 255 256 /* Both virtio and vhost 'send_pkt()' returns 'skb_len', 257 * but for reliability use 'ret' instead of 'skb_len'. 258 * Also if partial send happens (e.g. 'ret' != 'skb_len') 259 * somehow, we break this loop, but account such returned 260 * value in 'virtio_transport_put_credit()'. 261 */ 262 rest_len -= ret; 263 264 if (WARN_ONCE(ret != skb_len, 265 "'send_pkt()' returns %i, but %zu expected\n", 266 ret, skb_len)) 267 break; 268 } while (rest_len); 269 270 virtio_transport_put_credit(vvs, rest_len); 271 272 /* Return number of bytes, if any data has been sent. */ 273 if (rest_len != pkt_len) 274 ret = pkt_len - rest_len; 275 276 return ret; 277 } 278 279 static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 280 u32 len) 281 { 282 if (vvs->rx_bytes + len > vvs->buf_alloc) 283 return false; 284 285 vvs->rx_bytes += len; 286 return true; 287 } 288 289 static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 290 u32 len) 291 { 292 vvs->rx_bytes -= len; 293 vvs->fwd_cnt += len; 294 } 295 296 void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct sk_buff *skb) 297 { 298 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 299 300 spin_lock_bh(&vvs->rx_lock); 301 vvs->last_fwd_cnt = vvs->fwd_cnt; 302 hdr->fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 303 hdr->buf_alloc = cpu_to_le32(vvs->buf_alloc); 304 spin_unlock_bh(&vvs->rx_lock); 305 } 306 EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); 307 308 u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) 309 { 310 u32 ret; 311 312 if (!credit) 313 return 0; 314 315 spin_lock_bh(&vvs->tx_lock); 316 ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 317 if (ret > credit) 318 ret = credit; 319 vvs->tx_cnt += ret; 320 spin_unlock_bh(&vvs->tx_lock); 321 322 return ret; 323 } 324 EXPORT_SYMBOL_GPL(virtio_transport_get_credit); 325 326 void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) 327 { 328 if (!credit) 329 return; 330 331 spin_lock_bh(&vvs->tx_lock); 332 vvs->tx_cnt -= credit; 333 spin_unlock_bh(&vvs->tx_lock); 334 } 335 EXPORT_SYMBOL_GPL(virtio_transport_put_credit); 336 337 static int virtio_transport_send_credit_update(struct vsock_sock *vsk) 338 { 339 struct virtio_vsock_pkt_info info = { 340 .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, 341 .vsk = vsk, 342 }; 343 344 return virtio_transport_send_pkt_info(vsk, &info); 345 } 346 347 static ssize_t 348 virtio_transport_stream_do_peek(struct vsock_sock *vsk, 349 struct msghdr *msg, 350 size_t len) 351 { 352 struct virtio_vsock_sock *vvs = vsk->trans; 353 struct sk_buff *skb; 354 size_t total = 0; 355 int err; 356 357 spin_lock_bh(&vvs->rx_lock); 358 359 skb_queue_walk(&vvs->rx_queue, skb) { 360 size_t bytes; 361 362 bytes = len - total; 363 if (bytes > skb->len) 364 bytes = skb->len; 365 366 spin_unlock_bh(&vvs->rx_lock); 367 368 /* sk_lock is held by caller so no one else can dequeue. 369 * Unlock rx_lock since memcpy_to_msg() may sleep. 370 */ 371 err = memcpy_to_msg(msg, skb->data, bytes); 372 if (err) 373 goto out; 374 375 total += bytes; 376 377 spin_lock_bh(&vvs->rx_lock); 378 379 if (total == len) 380 break; 381 } 382 383 spin_unlock_bh(&vvs->rx_lock); 384 385 return total; 386 387 out: 388 if (total) 389 err = total; 390 return err; 391 } 392 393 static ssize_t 394 virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, 395 struct msghdr *msg, 396 size_t len) 397 { 398 struct virtio_vsock_sock *vvs = vsk->trans; 399 size_t bytes, total = 0; 400 struct sk_buff *skb; 401 int err = -EFAULT; 402 u32 free_space; 403 404 spin_lock_bh(&vvs->rx_lock); 405 406 if (WARN_ONCE(skb_queue_empty(&vvs->rx_queue) && vvs->rx_bytes, 407 "rx_queue is empty, but rx_bytes is non-zero\n")) { 408 spin_unlock_bh(&vvs->rx_lock); 409 return err; 410 } 411 412 while (total < len && !skb_queue_empty(&vvs->rx_queue)) { 413 skb = skb_peek(&vvs->rx_queue); 414 415 bytes = len - total; 416 if (bytes > skb->len) 417 bytes = skb->len; 418 419 /* sk_lock is held by caller so no one else can dequeue. 420 * Unlock rx_lock since memcpy_to_msg() may sleep. 421 */ 422 spin_unlock_bh(&vvs->rx_lock); 423 424 err = memcpy_to_msg(msg, skb->data, bytes); 425 if (err) 426 goto out; 427 428 spin_lock_bh(&vvs->rx_lock); 429 430 total += bytes; 431 skb_pull(skb, bytes); 432 433 if (skb->len == 0) { 434 u32 pkt_len = le32_to_cpu(virtio_vsock_hdr(skb)->len); 435 436 virtio_transport_dec_rx_pkt(vvs, pkt_len); 437 __skb_unlink(skb, &vvs->rx_queue); 438 consume_skb(skb); 439 } 440 } 441 442 free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); 443 444 spin_unlock_bh(&vvs->rx_lock); 445 446 /* To reduce the number of credit update messages, 447 * don't update credits as long as lots of space is available. 448 * Note: the limit chosen here is arbitrary. Setting the limit 449 * too high causes extra messages. Too low causes transmitter 450 * stalls. As stalls are in theory more expensive than extra 451 * messages, we set the limit to a high value. TODO: experiment 452 * with different values. 453 */ 454 if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) 455 virtio_transport_send_credit_update(vsk); 456 457 return total; 458 459 out: 460 if (total) 461 err = total; 462 return err; 463 } 464 465 static ssize_t 466 virtio_transport_seqpacket_do_peek(struct vsock_sock *vsk, 467 struct msghdr *msg) 468 { 469 struct virtio_vsock_sock *vvs = vsk->trans; 470 struct sk_buff *skb; 471 size_t total, len; 472 473 spin_lock_bh(&vvs->rx_lock); 474 475 if (!vvs->msg_count) { 476 spin_unlock_bh(&vvs->rx_lock); 477 return 0; 478 } 479 480 total = 0; 481 len = msg_data_left(msg); 482 483 skb_queue_walk(&vvs->rx_queue, skb) { 484 struct virtio_vsock_hdr *hdr; 485 486 if (total < len) { 487 size_t bytes; 488 int err; 489 490 bytes = len - total; 491 if (bytes > skb->len) 492 bytes = skb->len; 493 494 spin_unlock_bh(&vvs->rx_lock); 495 496 /* sk_lock is held by caller so no one else can dequeue. 497 * Unlock rx_lock since memcpy_to_msg() may sleep. 498 */ 499 err = memcpy_to_msg(msg, skb->data, bytes); 500 if (err) 501 return err; 502 503 spin_lock_bh(&vvs->rx_lock); 504 } 505 506 total += skb->len; 507 hdr = virtio_vsock_hdr(skb); 508 509 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 510 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) 511 msg->msg_flags |= MSG_EOR; 512 513 break; 514 } 515 } 516 517 spin_unlock_bh(&vvs->rx_lock); 518 519 return total; 520 } 521 522 static int virtio_transport_seqpacket_do_dequeue(struct vsock_sock *vsk, 523 struct msghdr *msg, 524 int flags) 525 { 526 struct virtio_vsock_sock *vvs = vsk->trans; 527 int dequeued_len = 0; 528 size_t user_buf_len = msg_data_left(msg); 529 bool msg_ready = false; 530 struct sk_buff *skb; 531 532 spin_lock_bh(&vvs->rx_lock); 533 534 if (vvs->msg_count == 0) { 535 spin_unlock_bh(&vvs->rx_lock); 536 return 0; 537 } 538 539 while (!msg_ready) { 540 struct virtio_vsock_hdr *hdr; 541 size_t pkt_len; 542 543 skb = __skb_dequeue(&vvs->rx_queue); 544 if (!skb) 545 break; 546 hdr = virtio_vsock_hdr(skb); 547 pkt_len = (size_t)le32_to_cpu(hdr->len); 548 549 if (dequeued_len >= 0) { 550 size_t bytes_to_copy; 551 552 bytes_to_copy = min(user_buf_len, pkt_len); 553 554 if (bytes_to_copy) { 555 int err; 556 557 /* sk_lock is held by caller so no one else can dequeue. 558 * Unlock rx_lock since memcpy_to_msg() may sleep. 559 */ 560 spin_unlock_bh(&vvs->rx_lock); 561 562 err = memcpy_to_msg(msg, skb->data, bytes_to_copy); 563 if (err) { 564 /* Copy of message failed. Rest of 565 * fragments will be freed without copy. 566 */ 567 dequeued_len = err; 568 } else { 569 user_buf_len -= bytes_to_copy; 570 } 571 572 spin_lock_bh(&vvs->rx_lock); 573 } 574 575 if (dequeued_len >= 0) 576 dequeued_len += pkt_len; 577 } 578 579 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) { 580 msg_ready = true; 581 vvs->msg_count--; 582 583 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOR) 584 msg->msg_flags |= MSG_EOR; 585 } 586 587 virtio_transport_dec_rx_pkt(vvs, pkt_len); 588 kfree_skb(skb); 589 } 590 591 spin_unlock_bh(&vvs->rx_lock); 592 593 virtio_transport_send_credit_update(vsk); 594 595 return dequeued_len; 596 } 597 598 ssize_t 599 virtio_transport_stream_dequeue(struct vsock_sock *vsk, 600 struct msghdr *msg, 601 size_t len, int flags) 602 { 603 if (flags & MSG_PEEK) 604 return virtio_transport_stream_do_peek(vsk, msg, len); 605 else 606 return virtio_transport_stream_do_dequeue(vsk, msg, len); 607 } 608 EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); 609 610 ssize_t 611 virtio_transport_seqpacket_dequeue(struct vsock_sock *vsk, 612 struct msghdr *msg, 613 int flags) 614 { 615 if (flags & MSG_PEEK) 616 return virtio_transport_seqpacket_do_peek(vsk, msg); 617 else 618 return virtio_transport_seqpacket_do_dequeue(vsk, msg, flags); 619 } 620 EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_dequeue); 621 622 int 623 virtio_transport_seqpacket_enqueue(struct vsock_sock *vsk, 624 struct msghdr *msg, 625 size_t len) 626 { 627 struct virtio_vsock_sock *vvs = vsk->trans; 628 629 spin_lock_bh(&vvs->tx_lock); 630 631 if (len > vvs->peer_buf_alloc) { 632 spin_unlock_bh(&vvs->tx_lock); 633 return -EMSGSIZE; 634 } 635 636 spin_unlock_bh(&vvs->tx_lock); 637 638 return virtio_transport_stream_enqueue(vsk, msg, len); 639 } 640 EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_enqueue); 641 642 int 643 virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 644 struct msghdr *msg, 645 size_t len, int flags) 646 { 647 return -EOPNOTSUPP; 648 } 649 EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); 650 651 s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) 652 { 653 struct virtio_vsock_sock *vvs = vsk->trans; 654 s64 bytes; 655 656 spin_lock_bh(&vvs->rx_lock); 657 bytes = vvs->rx_bytes; 658 spin_unlock_bh(&vvs->rx_lock); 659 660 return bytes; 661 } 662 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); 663 664 u32 virtio_transport_seqpacket_has_data(struct vsock_sock *vsk) 665 { 666 struct virtio_vsock_sock *vvs = vsk->trans; 667 u32 msg_count; 668 669 spin_lock_bh(&vvs->rx_lock); 670 msg_count = vvs->msg_count; 671 spin_unlock_bh(&vvs->rx_lock); 672 673 return msg_count; 674 } 675 EXPORT_SYMBOL_GPL(virtio_transport_seqpacket_has_data); 676 677 static s64 virtio_transport_has_space(struct vsock_sock *vsk) 678 { 679 struct virtio_vsock_sock *vvs = vsk->trans; 680 s64 bytes; 681 682 bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 683 if (bytes < 0) 684 bytes = 0; 685 686 return bytes; 687 } 688 689 s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) 690 { 691 struct virtio_vsock_sock *vvs = vsk->trans; 692 s64 bytes; 693 694 spin_lock_bh(&vvs->tx_lock); 695 bytes = virtio_transport_has_space(vsk); 696 spin_unlock_bh(&vvs->tx_lock); 697 698 return bytes; 699 } 700 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); 701 702 int virtio_transport_do_socket_init(struct vsock_sock *vsk, 703 struct vsock_sock *psk) 704 { 705 struct virtio_vsock_sock *vvs; 706 707 vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); 708 if (!vvs) 709 return -ENOMEM; 710 711 vsk->trans = vvs; 712 vvs->vsk = vsk; 713 if (psk && psk->trans) { 714 struct virtio_vsock_sock *ptrans = psk->trans; 715 716 vvs->peer_buf_alloc = ptrans->peer_buf_alloc; 717 } 718 719 if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE) 720 vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE; 721 722 vvs->buf_alloc = vsk->buffer_size; 723 724 spin_lock_init(&vvs->rx_lock); 725 spin_lock_init(&vvs->tx_lock); 726 skb_queue_head_init(&vvs->rx_queue); 727 728 return 0; 729 } 730 EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); 731 732 /* sk_lock held by the caller */ 733 void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val) 734 { 735 struct virtio_vsock_sock *vvs = vsk->trans; 736 737 if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE) 738 *val = VIRTIO_VSOCK_MAX_BUF_SIZE; 739 740 vvs->buf_alloc = *val; 741 742 virtio_transport_send_credit_update(vsk); 743 } 744 EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size); 745 746 int 747 virtio_transport_notify_poll_in(struct vsock_sock *vsk, 748 size_t target, 749 bool *data_ready_now) 750 { 751 *data_ready_now = vsock_stream_has_data(vsk) >= target; 752 753 return 0; 754 } 755 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); 756 757 int 758 virtio_transport_notify_poll_out(struct vsock_sock *vsk, 759 size_t target, 760 bool *space_avail_now) 761 { 762 s64 free_space; 763 764 free_space = vsock_stream_has_space(vsk); 765 if (free_space > 0) 766 *space_avail_now = true; 767 else if (free_space == 0) 768 *space_avail_now = false; 769 770 return 0; 771 } 772 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); 773 774 int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 775 size_t target, struct vsock_transport_recv_notify_data *data) 776 { 777 return 0; 778 } 779 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); 780 781 int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 782 size_t target, struct vsock_transport_recv_notify_data *data) 783 { 784 return 0; 785 } 786 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); 787 788 int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 789 size_t target, struct vsock_transport_recv_notify_data *data) 790 { 791 return 0; 792 } 793 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); 794 795 int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 796 size_t target, ssize_t copied, bool data_read, 797 struct vsock_transport_recv_notify_data *data) 798 { 799 return 0; 800 } 801 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); 802 803 int virtio_transport_notify_send_init(struct vsock_sock *vsk, 804 struct vsock_transport_send_notify_data *data) 805 { 806 return 0; 807 } 808 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); 809 810 int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 811 struct vsock_transport_send_notify_data *data) 812 { 813 return 0; 814 } 815 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); 816 817 int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 818 struct vsock_transport_send_notify_data *data) 819 { 820 return 0; 821 } 822 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); 823 824 int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 825 ssize_t written, struct vsock_transport_send_notify_data *data) 826 { 827 return 0; 828 } 829 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); 830 831 u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) 832 { 833 return vsk->buffer_size; 834 } 835 EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); 836 837 bool virtio_transport_stream_is_active(struct vsock_sock *vsk) 838 { 839 return true; 840 } 841 EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); 842 843 bool virtio_transport_stream_allow(u32 cid, u32 port) 844 { 845 return true; 846 } 847 EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); 848 849 int virtio_transport_dgram_bind(struct vsock_sock *vsk, 850 struct sockaddr_vm *addr) 851 { 852 return -EOPNOTSUPP; 853 } 854 EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); 855 856 bool virtio_transport_dgram_allow(u32 cid, u32 port) 857 { 858 return false; 859 } 860 EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); 861 862 int virtio_transport_connect(struct vsock_sock *vsk) 863 { 864 struct virtio_vsock_pkt_info info = { 865 .op = VIRTIO_VSOCK_OP_REQUEST, 866 .vsk = vsk, 867 }; 868 869 return virtio_transport_send_pkt_info(vsk, &info); 870 } 871 EXPORT_SYMBOL_GPL(virtio_transport_connect); 872 873 int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) 874 { 875 struct virtio_vsock_pkt_info info = { 876 .op = VIRTIO_VSOCK_OP_SHUTDOWN, 877 .flags = (mode & RCV_SHUTDOWN ? 878 VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | 879 (mode & SEND_SHUTDOWN ? 880 VIRTIO_VSOCK_SHUTDOWN_SEND : 0), 881 .vsk = vsk, 882 }; 883 884 return virtio_transport_send_pkt_info(vsk, &info); 885 } 886 EXPORT_SYMBOL_GPL(virtio_transport_shutdown); 887 888 int 889 virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 890 struct sockaddr_vm *remote_addr, 891 struct msghdr *msg, 892 size_t dgram_len) 893 { 894 return -EOPNOTSUPP; 895 } 896 EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); 897 898 ssize_t 899 virtio_transport_stream_enqueue(struct vsock_sock *vsk, 900 struct msghdr *msg, 901 size_t len) 902 { 903 struct virtio_vsock_pkt_info info = { 904 .op = VIRTIO_VSOCK_OP_RW, 905 .msg = msg, 906 .pkt_len = len, 907 .vsk = vsk, 908 }; 909 910 return virtio_transport_send_pkt_info(vsk, &info); 911 } 912 EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); 913 914 void virtio_transport_destruct(struct vsock_sock *vsk) 915 { 916 struct virtio_vsock_sock *vvs = vsk->trans; 917 918 kfree(vvs); 919 } 920 EXPORT_SYMBOL_GPL(virtio_transport_destruct); 921 922 static int virtio_transport_reset(struct vsock_sock *vsk, 923 struct sk_buff *skb) 924 { 925 struct virtio_vsock_pkt_info info = { 926 .op = VIRTIO_VSOCK_OP_RST, 927 .reply = !!skb, 928 .vsk = vsk, 929 }; 930 931 /* Send RST only if the original pkt is not a RST pkt */ 932 if (skb && le16_to_cpu(virtio_vsock_hdr(skb)->op) == VIRTIO_VSOCK_OP_RST) 933 return 0; 934 935 return virtio_transport_send_pkt_info(vsk, &info); 936 } 937 938 /* Normally packets are associated with a socket. There may be no socket if an 939 * attempt was made to connect to a socket that does not exist. 940 */ 941 static int virtio_transport_reset_no_sock(const struct virtio_transport *t, 942 struct sk_buff *skb) 943 { 944 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 945 struct virtio_vsock_pkt_info info = { 946 .op = VIRTIO_VSOCK_OP_RST, 947 .type = le16_to_cpu(hdr->type), 948 .reply = true, 949 }; 950 struct sk_buff *reply; 951 952 /* Send RST only if the original pkt is not a RST pkt */ 953 if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) 954 return 0; 955 956 if (!t) 957 return -ENOTCONN; 958 959 reply = virtio_transport_alloc_skb(&info, 0, 960 le64_to_cpu(hdr->dst_cid), 961 le32_to_cpu(hdr->dst_port), 962 le64_to_cpu(hdr->src_cid), 963 le32_to_cpu(hdr->src_port)); 964 if (!reply) 965 return -ENOMEM; 966 967 return t->send_pkt(reply); 968 } 969 970 /* This function should be called with sk_lock held and SOCK_DONE set */ 971 static void virtio_transport_remove_sock(struct vsock_sock *vsk) 972 { 973 struct virtio_vsock_sock *vvs = vsk->trans; 974 975 /* We don't need to take rx_lock, as the socket is closing and we are 976 * removing it. 977 */ 978 __skb_queue_purge(&vvs->rx_queue); 979 vsock_remove_sock(vsk); 980 } 981 982 static void virtio_transport_wait_close(struct sock *sk, long timeout) 983 { 984 if (timeout) { 985 DEFINE_WAIT_FUNC(wait, woken_wake_function); 986 987 add_wait_queue(sk_sleep(sk), &wait); 988 989 do { 990 if (sk_wait_event(sk, &timeout, 991 sock_flag(sk, SOCK_DONE), &wait)) 992 break; 993 } while (!signal_pending(current) && timeout); 994 995 remove_wait_queue(sk_sleep(sk), &wait); 996 } 997 } 998 999 static void virtio_transport_do_close(struct vsock_sock *vsk, 1000 bool cancel_timeout) 1001 { 1002 struct sock *sk = sk_vsock(vsk); 1003 1004 sock_set_flag(sk, SOCK_DONE); 1005 vsk->peer_shutdown = SHUTDOWN_MASK; 1006 if (vsock_stream_has_data(vsk) <= 0) 1007 sk->sk_state = TCP_CLOSING; 1008 sk->sk_state_change(sk); 1009 1010 if (vsk->close_work_scheduled && 1011 (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { 1012 vsk->close_work_scheduled = false; 1013 1014 virtio_transport_remove_sock(vsk); 1015 1016 /* Release refcnt obtained when we scheduled the timeout */ 1017 sock_put(sk); 1018 } 1019 } 1020 1021 static void virtio_transport_close_timeout(struct work_struct *work) 1022 { 1023 struct vsock_sock *vsk = 1024 container_of(work, struct vsock_sock, close_work.work); 1025 struct sock *sk = sk_vsock(vsk); 1026 1027 sock_hold(sk); 1028 lock_sock(sk); 1029 1030 if (!sock_flag(sk, SOCK_DONE)) { 1031 (void)virtio_transport_reset(vsk, NULL); 1032 1033 virtio_transport_do_close(vsk, false); 1034 } 1035 1036 vsk->close_work_scheduled = false; 1037 1038 release_sock(sk); 1039 sock_put(sk); 1040 } 1041 1042 /* User context, vsk->sk is locked */ 1043 static bool virtio_transport_close(struct vsock_sock *vsk) 1044 { 1045 struct sock *sk = &vsk->sk; 1046 1047 if (!(sk->sk_state == TCP_ESTABLISHED || 1048 sk->sk_state == TCP_CLOSING)) 1049 return true; 1050 1051 /* Already received SHUTDOWN from peer, reply with RST */ 1052 if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { 1053 (void)virtio_transport_reset(vsk, NULL); 1054 return true; 1055 } 1056 1057 if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) 1058 (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); 1059 1060 if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) 1061 virtio_transport_wait_close(sk, sk->sk_lingertime); 1062 1063 if (sock_flag(sk, SOCK_DONE)) { 1064 return true; 1065 } 1066 1067 sock_hold(sk); 1068 INIT_DELAYED_WORK(&vsk->close_work, 1069 virtio_transport_close_timeout); 1070 vsk->close_work_scheduled = true; 1071 schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); 1072 return false; 1073 } 1074 1075 void virtio_transport_release(struct vsock_sock *vsk) 1076 { 1077 struct sock *sk = &vsk->sk; 1078 bool remove_sock = true; 1079 1080 if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) 1081 remove_sock = virtio_transport_close(vsk); 1082 1083 if (remove_sock) { 1084 sock_set_flag(sk, SOCK_DONE); 1085 virtio_transport_remove_sock(vsk); 1086 } 1087 } 1088 EXPORT_SYMBOL_GPL(virtio_transport_release); 1089 1090 static int 1091 virtio_transport_recv_connecting(struct sock *sk, 1092 struct sk_buff *skb) 1093 { 1094 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1095 struct vsock_sock *vsk = vsock_sk(sk); 1096 int skerr; 1097 int err; 1098 1099 switch (le16_to_cpu(hdr->op)) { 1100 case VIRTIO_VSOCK_OP_RESPONSE: 1101 sk->sk_state = TCP_ESTABLISHED; 1102 sk->sk_socket->state = SS_CONNECTED; 1103 vsock_insert_connected(vsk); 1104 sk->sk_state_change(sk); 1105 break; 1106 case VIRTIO_VSOCK_OP_INVALID: 1107 break; 1108 case VIRTIO_VSOCK_OP_RST: 1109 skerr = ECONNRESET; 1110 err = 0; 1111 goto destroy; 1112 default: 1113 skerr = EPROTO; 1114 err = -EINVAL; 1115 goto destroy; 1116 } 1117 return 0; 1118 1119 destroy: 1120 virtio_transport_reset(vsk, skb); 1121 sk->sk_state = TCP_CLOSE; 1122 sk->sk_err = skerr; 1123 sk_error_report(sk); 1124 return err; 1125 } 1126 1127 static void 1128 virtio_transport_recv_enqueue(struct vsock_sock *vsk, 1129 struct sk_buff *skb) 1130 { 1131 struct virtio_vsock_sock *vvs = vsk->trans; 1132 bool can_enqueue, free_pkt = false; 1133 struct virtio_vsock_hdr *hdr; 1134 u32 len; 1135 1136 hdr = virtio_vsock_hdr(skb); 1137 len = le32_to_cpu(hdr->len); 1138 1139 spin_lock_bh(&vvs->rx_lock); 1140 1141 can_enqueue = virtio_transport_inc_rx_pkt(vvs, len); 1142 if (!can_enqueue) { 1143 free_pkt = true; 1144 goto out; 1145 } 1146 1147 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SEQ_EOM) 1148 vvs->msg_count++; 1149 1150 /* Try to copy small packets into the buffer of last packet queued, 1151 * to avoid wasting memory queueing the entire buffer with a small 1152 * payload. 1153 */ 1154 if (len <= GOOD_COPY_LEN && !skb_queue_empty(&vvs->rx_queue)) { 1155 struct virtio_vsock_hdr *last_hdr; 1156 struct sk_buff *last_skb; 1157 1158 last_skb = skb_peek_tail(&vvs->rx_queue); 1159 last_hdr = virtio_vsock_hdr(last_skb); 1160 1161 /* If there is space in the last packet queued, we copy the 1162 * new packet in its buffer. We avoid this if the last packet 1163 * queued has VIRTIO_VSOCK_SEQ_EOM set, because this is 1164 * delimiter of SEQPACKET message, so 'pkt' is the first packet 1165 * of a new message. 1166 */ 1167 if (skb->len < skb_tailroom(last_skb) && 1168 !(le32_to_cpu(last_hdr->flags) & VIRTIO_VSOCK_SEQ_EOM)) { 1169 memcpy(skb_put(last_skb, skb->len), skb->data, skb->len); 1170 free_pkt = true; 1171 last_hdr->flags |= hdr->flags; 1172 le32_add_cpu(&last_hdr->len, len); 1173 goto out; 1174 } 1175 } 1176 1177 __skb_queue_tail(&vvs->rx_queue, skb); 1178 1179 out: 1180 spin_unlock_bh(&vvs->rx_lock); 1181 if (free_pkt) 1182 kfree_skb(skb); 1183 } 1184 1185 static int 1186 virtio_transport_recv_connected(struct sock *sk, 1187 struct sk_buff *skb) 1188 { 1189 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1190 struct vsock_sock *vsk = vsock_sk(sk); 1191 int err = 0; 1192 1193 switch (le16_to_cpu(hdr->op)) { 1194 case VIRTIO_VSOCK_OP_RW: 1195 virtio_transport_recv_enqueue(vsk, skb); 1196 vsock_data_ready(sk); 1197 return err; 1198 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: 1199 virtio_transport_send_credit_update(vsk); 1200 break; 1201 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 1202 sk->sk_write_space(sk); 1203 break; 1204 case VIRTIO_VSOCK_OP_SHUTDOWN: 1205 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 1206 vsk->peer_shutdown |= RCV_SHUTDOWN; 1207 if (le32_to_cpu(hdr->flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 1208 vsk->peer_shutdown |= SEND_SHUTDOWN; 1209 if (vsk->peer_shutdown == SHUTDOWN_MASK) { 1210 if (vsock_stream_has_data(vsk) <= 0 && !sock_flag(sk, SOCK_DONE)) { 1211 (void)virtio_transport_reset(vsk, NULL); 1212 virtio_transport_do_close(vsk, true); 1213 } 1214 /* Remove this socket anyway because the remote peer sent 1215 * the shutdown. This way a new connection will succeed 1216 * if the remote peer uses the same source port, 1217 * even if the old socket is still unreleased, but now disconnected. 1218 */ 1219 vsock_remove_sock(vsk); 1220 } 1221 if (le32_to_cpu(virtio_vsock_hdr(skb)->flags)) 1222 sk->sk_state_change(sk); 1223 break; 1224 case VIRTIO_VSOCK_OP_RST: 1225 virtio_transport_do_close(vsk, true); 1226 break; 1227 default: 1228 err = -EINVAL; 1229 break; 1230 } 1231 1232 kfree_skb(skb); 1233 return err; 1234 } 1235 1236 static void 1237 virtio_transport_recv_disconnecting(struct sock *sk, 1238 struct sk_buff *skb) 1239 { 1240 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1241 struct vsock_sock *vsk = vsock_sk(sk); 1242 1243 if (le16_to_cpu(hdr->op) == VIRTIO_VSOCK_OP_RST) 1244 virtio_transport_do_close(vsk, true); 1245 } 1246 1247 static int 1248 virtio_transport_send_response(struct vsock_sock *vsk, 1249 struct sk_buff *skb) 1250 { 1251 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1252 struct virtio_vsock_pkt_info info = { 1253 .op = VIRTIO_VSOCK_OP_RESPONSE, 1254 .remote_cid = le64_to_cpu(hdr->src_cid), 1255 .remote_port = le32_to_cpu(hdr->src_port), 1256 .reply = true, 1257 .vsk = vsk, 1258 }; 1259 1260 return virtio_transport_send_pkt_info(vsk, &info); 1261 } 1262 1263 static bool virtio_transport_space_update(struct sock *sk, 1264 struct sk_buff *skb) 1265 { 1266 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1267 struct vsock_sock *vsk = vsock_sk(sk); 1268 struct virtio_vsock_sock *vvs = vsk->trans; 1269 bool space_available; 1270 1271 /* Listener sockets are not associated with any transport, so we are 1272 * not able to take the state to see if there is space available in the 1273 * remote peer, but since they are only used to receive requests, we 1274 * can assume that there is always space available in the other peer. 1275 */ 1276 if (!vvs) 1277 return true; 1278 1279 /* buf_alloc and fwd_cnt is always included in the hdr */ 1280 spin_lock_bh(&vvs->tx_lock); 1281 vvs->peer_buf_alloc = le32_to_cpu(hdr->buf_alloc); 1282 vvs->peer_fwd_cnt = le32_to_cpu(hdr->fwd_cnt); 1283 space_available = virtio_transport_has_space(vsk); 1284 spin_unlock_bh(&vvs->tx_lock); 1285 return space_available; 1286 } 1287 1288 /* Handle server socket */ 1289 static int 1290 virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb, 1291 struct virtio_transport *t) 1292 { 1293 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1294 struct vsock_sock *vsk = vsock_sk(sk); 1295 struct vsock_sock *vchild; 1296 struct sock *child; 1297 int ret; 1298 1299 if (le16_to_cpu(hdr->op) != VIRTIO_VSOCK_OP_REQUEST) { 1300 virtio_transport_reset_no_sock(t, skb); 1301 return -EINVAL; 1302 } 1303 1304 if (sk_acceptq_is_full(sk)) { 1305 virtio_transport_reset_no_sock(t, skb); 1306 return -ENOMEM; 1307 } 1308 1309 child = vsock_create_connected(sk); 1310 if (!child) { 1311 virtio_transport_reset_no_sock(t, skb); 1312 return -ENOMEM; 1313 } 1314 1315 sk_acceptq_added(sk); 1316 1317 lock_sock_nested(child, SINGLE_DEPTH_NESTING); 1318 1319 child->sk_state = TCP_ESTABLISHED; 1320 1321 vchild = vsock_sk(child); 1322 vsock_addr_init(&vchild->local_addr, le64_to_cpu(hdr->dst_cid), 1323 le32_to_cpu(hdr->dst_port)); 1324 vsock_addr_init(&vchild->remote_addr, le64_to_cpu(hdr->src_cid), 1325 le32_to_cpu(hdr->src_port)); 1326 1327 ret = vsock_assign_transport(vchild, vsk); 1328 /* Transport assigned (looking at remote_addr) must be the same 1329 * where we received the request. 1330 */ 1331 if (ret || vchild->transport != &t->transport) { 1332 release_sock(child); 1333 virtio_transport_reset_no_sock(t, skb); 1334 sock_put(child); 1335 return ret; 1336 } 1337 1338 if (virtio_transport_space_update(child, skb)) 1339 child->sk_write_space(child); 1340 1341 vsock_insert_connected(vchild); 1342 vsock_enqueue_accept(sk, child); 1343 virtio_transport_send_response(vchild, skb); 1344 1345 release_sock(child); 1346 1347 sk->sk_data_ready(sk); 1348 return 0; 1349 } 1350 1351 static bool virtio_transport_valid_type(u16 type) 1352 { 1353 return (type == VIRTIO_VSOCK_TYPE_STREAM) || 1354 (type == VIRTIO_VSOCK_TYPE_SEQPACKET); 1355 } 1356 1357 /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex 1358 * lock. 1359 */ 1360 void virtio_transport_recv_pkt(struct virtio_transport *t, 1361 struct sk_buff *skb) 1362 { 1363 struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); 1364 struct sockaddr_vm src, dst; 1365 struct vsock_sock *vsk; 1366 struct sock *sk; 1367 bool space_available; 1368 1369 vsock_addr_init(&src, le64_to_cpu(hdr->src_cid), 1370 le32_to_cpu(hdr->src_port)); 1371 vsock_addr_init(&dst, le64_to_cpu(hdr->dst_cid), 1372 le32_to_cpu(hdr->dst_port)); 1373 1374 trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 1375 dst.svm_cid, dst.svm_port, 1376 le32_to_cpu(hdr->len), 1377 le16_to_cpu(hdr->type), 1378 le16_to_cpu(hdr->op), 1379 le32_to_cpu(hdr->flags), 1380 le32_to_cpu(hdr->buf_alloc), 1381 le32_to_cpu(hdr->fwd_cnt)); 1382 1383 if (!virtio_transport_valid_type(le16_to_cpu(hdr->type))) { 1384 (void)virtio_transport_reset_no_sock(t, skb); 1385 goto free_pkt; 1386 } 1387 1388 /* The socket must be in connected or bound table 1389 * otherwise send reset back 1390 */ 1391 sk = vsock_find_connected_socket(&src, &dst); 1392 if (!sk) { 1393 sk = vsock_find_bound_socket(&dst); 1394 if (!sk) { 1395 (void)virtio_transport_reset_no_sock(t, skb); 1396 goto free_pkt; 1397 } 1398 } 1399 1400 if (virtio_transport_get_type(sk) != le16_to_cpu(hdr->type)) { 1401 (void)virtio_transport_reset_no_sock(t, skb); 1402 sock_put(sk); 1403 goto free_pkt; 1404 } 1405 1406 if (!skb_set_owner_sk_safe(skb, sk)) { 1407 WARN_ONCE(1, "receiving vsock socket has sk_refcnt == 0\n"); 1408 goto free_pkt; 1409 } 1410 1411 vsk = vsock_sk(sk); 1412 1413 lock_sock(sk); 1414 1415 /* Check if sk has been closed before lock_sock */ 1416 if (sock_flag(sk, SOCK_DONE)) { 1417 (void)virtio_transport_reset_no_sock(t, skb); 1418 release_sock(sk); 1419 sock_put(sk); 1420 goto free_pkt; 1421 } 1422 1423 space_available = virtio_transport_space_update(sk, skb); 1424 1425 /* Update CID in case it has changed after a transport reset event */ 1426 if (vsk->local_addr.svm_cid != VMADDR_CID_ANY) 1427 vsk->local_addr.svm_cid = dst.svm_cid; 1428 1429 if (space_available) 1430 sk->sk_write_space(sk); 1431 1432 switch (sk->sk_state) { 1433 case TCP_LISTEN: 1434 virtio_transport_recv_listen(sk, skb, t); 1435 kfree_skb(skb); 1436 break; 1437 case TCP_SYN_SENT: 1438 virtio_transport_recv_connecting(sk, skb); 1439 kfree_skb(skb); 1440 break; 1441 case TCP_ESTABLISHED: 1442 virtio_transport_recv_connected(sk, skb); 1443 break; 1444 case TCP_CLOSING: 1445 virtio_transport_recv_disconnecting(sk, skb); 1446 kfree_skb(skb); 1447 break; 1448 default: 1449 (void)virtio_transport_reset_no_sock(t, skb); 1450 kfree_skb(skb); 1451 break; 1452 } 1453 1454 release_sock(sk); 1455 1456 /* Release refcnt obtained when we fetched this socket out of the 1457 * bound or connected list. 1458 */ 1459 sock_put(sk); 1460 return; 1461 1462 free_pkt: 1463 kfree_skb(skb); 1464 } 1465 EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 1466 1467 /* Remove skbs found in a queue that have a vsk that matches. 1468 * 1469 * Each skb is freed. 1470 * 1471 * Returns the count of skbs that were reply packets. 1472 */ 1473 int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *queue) 1474 { 1475 struct sk_buff_head freeme; 1476 struct sk_buff *skb, *tmp; 1477 int cnt = 0; 1478 1479 skb_queue_head_init(&freeme); 1480 1481 spin_lock_bh(&queue->lock); 1482 skb_queue_walk_safe(queue, skb, tmp) { 1483 if (vsock_sk(skb->sk) != vsk) 1484 continue; 1485 1486 __skb_unlink(skb, queue); 1487 __skb_queue_tail(&freeme, skb); 1488 1489 if (virtio_vsock_skb_reply(skb)) 1490 cnt++; 1491 } 1492 spin_unlock_bh(&queue->lock); 1493 1494 __skb_queue_purge(&freeme); 1495 1496 return cnt; 1497 } 1498 EXPORT_SYMBOL_GPL(virtio_transport_purge_skbs); 1499 1500 int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_actor) 1501 { 1502 struct virtio_vsock_sock *vvs = vsk->trans; 1503 struct sock *sk = sk_vsock(vsk); 1504 struct sk_buff *skb; 1505 int off = 0; 1506 int err; 1507 1508 spin_lock_bh(&vvs->rx_lock); 1509 /* Use __skb_recv_datagram() for race-free handling of the receive. It 1510 * works for types other than dgrams. 1511 */ 1512 skb = __skb_recv_datagram(sk, &vvs->rx_queue, MSG_DONTWAIT, &off, &err); 1513 spin_unlock_bh(&vvs->rx_lock); 1514 1515 if (!skb) 1516 return err; 1517 1518 return recv_actor(sk, skb); 1519 } 1520 EXPORT_SYMBOL_GPL(virtio_transport_read_skb); 1521 1522 MODULE_LICENSE("GPL v2"); 1523 MODULE_AUTHOR("Asias He"); 1524 MODULE_DESCRIPTION("common code for virtio vsock"); 1525