1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * common code for virtio vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/spinlock.h> 10 #include <linux/module.h> 11 #include <linux/sched/signal.h> 12 #include <linux/ctype.h> 13 #include <linux/list.h> 14 #include <linux/virtio_vsock.h> 15 #include <uapi/linux/vsockmon.h> 16 17 #include <net/sock.h> 18 #include <net/af_vsock.h> 19 20 #define CREATE_TRACE_POINTS 21 #include <trace/events/vsock_virtio_transport_common.h> 22 23 /* How long to wait for graceful shutdown of a connection */ 24 #define VSOCK_CLOSE_TIMEOUT (8 * HZ) 25 26 /* Threshold for detecting small packets to copy */ 27 #define GOOD_COPY_LEN 128 28 29 static const struct virtio_transport * 30 virtio_transport_get_ops(struct vsock_sock *vsk) 31 { 32 const struct vsock_transport *t = vsock_core_get_transport(vsk); 33 34 if (WARN_ON(!t)) 35 return NULL; 36 37 return container_of(t, struct virtio_transport, transport); 38 } 39 40 static struct virtio_vsock_pkt * 41 virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info, 42 size_t len, 43 u32 src_cid, 44 u32 src_port, 45 u32 dst_cid, 46 u32 dst_port) 47 { 48 struct virtio_vsock_pkt *pkt; 49 int err; 50 51 pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 52 if (!pkt) 53 return NULL; 54 55 pkt->hdr.type = cpu_to_le16(info->type); 56 pkt->hdr.op = cpu_to_le16(info->op); 57 pkt->hdr.src_cid = cpu_to_le64(src_cid); 58 pkt->hdr.dst_cid = cpu_to_le64(dst_cid); 59 pkt->hdr.src_port = cpu_to_le32(src_port); 60 pkt->hdr.dst_port = cpu_to_le32(dst_port); 61 pkt->hdr.flags = cpu_to_le32(info->flags); 62 pkt->len = len; 63 pkt->hdr.len = cpu_to_le32(len); 64 pkt->reply = info->reply; 65 pkt->vsk = info->vsk; 66 67 if (info->msg && len > 0) { 68 pkt->buf = kmalloc(len, GFP_KERNEL); 69 if (!pkt->buf) 70 goto out_pkt; 71 72 pkt->buf_len = len; 73 74 err = memcpy_from_msg(pkt->buf, info->msg, len); 75 if (err) 76 goto out; 77 } 78 79 trace_virtio_transport_alloc_pkt(src_cid, src_port, 80 dst_cid, dst_port, 81 len, 82 info->type, 83 info->op, 84 info->flags); 85 86 return pkt; 87 88 out: 89 kfree(pkt->buf); 90 out_pkt: 91 kfree(pkt); 92 return NULL; 93 } 94 95 /* Packet capture */ 96 static struct sk_buff *virtio_transport_build_skb(void *opaque) 97 { 98 struct virtio_vsock_pkt *pkt = opaque; 99 struct af_vsockmon_hdr *hdr; 100 struct sk_buff *skb; 101 size_t payload_len; 102 void *payload_buf; 103 104 /* A packet could be split to fit the RX buffer, so we can retrieve 105 * the payload length from the header and the buffer pointer taking 106 * care of the offset in the original packet. 107 */ 108 payload_len = le32_to_cpu(pkt->hdr.len); 109 payload_buf = pkt->buf + pkt->off; 110 111 skb = alloc_skb(sizeof(*hdr) + sizeof(pkt->hdr) + payload_len, 112 GFP_ATOMIC); 113 if (!skb) 114 return NULL; 115 116 hdr = skb_put(skb, sizeof(*hdr)); 117 118 /* pkt->hdr is little-endian so no need to byteswap here */ 119 hdr->src_cid = pkt->hdr.src_cid; 120 hdr->src_port = pkt->hdr.src_port; 121 hdr->dst_cid = pkt->hdr.dst_cid; 122 hdr->dst_port = pkt->hdr.dst_port; 123 124 hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); 125 hdr->len = cpu_to_le16(sizeof(pkt->hdr)); 126 memset(hdr->reserved, 0, sizeof(hdr->reserved)); 127 128 switch (le16_to_cpu(pkt->hdr.op)) { 129 case VIRTIO_VSOCK_OP_REQUEST: 130 case VIRTIO_VSOCK_OP_RESPONSE: 131 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); 132 break; 133 case VIRTIO_VSOCK_OP_RST: 134 case VIRTIO_VSOCK_OP_SHUTDOWN: 135 hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT); 136 break; 137 case VIRTIO_VSOCK_OP_RW: 138 hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD); 139 break; 140 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 141 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: 142 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL); 143 break; 144 default: 145 hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN); 146 break; 147 } 148 149 skb_put_data(skb, &pkt->hdr, sizeof(pkt->hdr)); 150 151 if (payload_len) { 152 skb_put_data(skb, payload_buf, payload_len); 153 } 154 155 return skb; 156 } 157 158 void virtio_transport_deliver_tap_pkt(struct virtio_vsock_pkt *pkt) 159 { 160 if (pkt->tap_delivered) 161 return; 162 163 vsock_deliver_tap(virtio_transport_build_skb, pkt); 164 pkt->tap_delivered = true; 165 } 166 EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); 167 168 /* This function can only be used on connecting/connected sockets, 169 * since a socket assigned to a transport is required. 170 * 171 * Do not use on listener sockets! 172 */ 173 static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, 174 struct virtio_vsock_pkt_info *info) 175 { 176 u32 src_cid, src_port, dst_cid, dst_port; 177 const struct virtio_transport *t_ops; 178 struct virtio_vsock_sock *vvs; 179 struct virtio_vsock_pkt *pkt; 180 u32 pkt_len = info->pkt_len; 181 182 t_ops = virtio_transport_get_ops(vsk); 183 if (unlikely(!t_ops)) 184 return -EFAULT; 185 186 src_cid = t_ops->transport.get_local_cid(); 187 src_port = vsk->local_addr.svm_port; 188 if (!info->remote_cid) { 189 dst_cid = vsk->remote_addr.svm_cid; 190 dst_port = vsk->remote_addr.svm_port; 191 } else { 192 dst_cid = info->remote_cid; 193 dst_port = info->remote_port; 194 } 195 196 vvs = vsk->trans; 197 198 /* we can send less than pkt_len bytes */ 199 if (pkt_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) 200 pkt_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; 201 202 /* virtio_transport_get_credit might return less than pkt_len credit */ 203 pkt_len = virtio_transport_get_credit(vvs, pkt_len); 204 205 /* Do not send zero length OP_RW pkt */ 206 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 207 return pkt_len; 208 209 pkt = virtio_transport_alloc_pkt(info, pkt_len, 210 src_cid, src_port, 211 dst_cid, dst_port); 212 if (!pkt) { 213 virtio_transport_put_credit(vvs, pkt_len); 214 return -ENOMEM; 215 } 216 217 virtio_transport_inc_tx_pkt(vvs, pkt); 218 219 return t_ops->send_pkt(pkt); 220 } 221 222 static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 223 struct virtio_vsock_pkt *pkt) 224 { 225 if (vvs->rx_bytes + pkt->len > vvs->buf_alloc) 226 return false; 227 228 vvs->rx_bytes += pkt->len; 229 return true; 230 } 231 232 static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 233 struct virtio_vsock_pkt *pkt) 234 { 235 vvs->rx_bytes -= pkt->len; 236 vvs->fwd_cnt += pkt->len; 237 } 238 239 void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) 240 { 241 spin_lock_bh(&vvs->rx_lock); 242 vvs->last_fwd_cnt = vvs->fwd_cnt; 243 pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 244 pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); 245 spin_unlock_bh(&vvs->rx_lock); 246 } 247 EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); 248 249 u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) 250 { 251 u32 ret; 252 253 spin_lock_bh(&vvs->tx_lock); 254 ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 255 if (ret > credit) 256 ret = credit; 257 vvs->tx_cnt += ret; 258 spin_unlock_bh(&vvs->tx_lock); 259 260 return ret; 261 } 262 EXPORT_SYMBOL_GPL(virtio_transport_get_credit); 263 264 void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) 265 { 266 spin_lock_bh(&vvs->tx_lock); 267 vvs->tx_cnt -= credit; 268 spin_unlock_bh(&vvs->tx_lock); 269 } 270 EXPORT_SYMBOL_GPL(virtio_transport_put_credit); 271 272 static int virtio_transport_send_credit_update(struct vsock_sock *vsk, 273 int type, 274 struct virtio_vsock_hdr *hdr) 275 { 276 struct virtio_vsock_pkt_info info = { 277 .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, 278 .type = type, 279 .vsk = vsk, 280 }; 281 282 return virtio_transport_send_pkt_info(vsk, &info); 283 } 284 285 static ssize_t 286 virtio_transport_stream_do_peek(struct vsock_sock *vsk, 287 struct msghdr *msg, 288 size_t len) 289 { 290 struct virtio_vsock_sock *vvs = vsk->trans; 291 struct virtio_vsock_pkt *pkt; 292 size_t bytes, total = 0, off; 293 int err = -EFAULT; 294 295 spin_lock_bh(&vvs->rx_lock); 296 297 list_for_each_entry(pkt, &vvs->rx_queue, list) { 298 off = pkt->off; 299 300 if (total == len) 301 break; 302 303 while (total < len && off < pkt->len) { 304 bytes = len - total; 305 if (bytes > pkt->len - off) 306 bytes = pkt->len - off; 307 308 /* sk_lock is held by caller so no one else can dequeue. 309 * Unlock rx_lock since memcpy_to_msg() may sleep. 310 */ 311 spin_unlock_bh(&vvs->rx_lock); 312 313 err = memcpy_to_msg(msg, pkt->buf + off, bytes); 314 if (err) 315 goto out; 316 317 spin_lock_bh(&vvs->rx_lock); 318 319 total += bytes; 320 off += bytes; 321 } 322 } 323 324 spin_unlock_bh(&vvs->rx_lock); 325 326 return total; 327 328 out: 329 if (total) 330 err = total; 331 return err; 332 } 333 334 static ssize_t 335 virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, 336 struct msghdr *msg, 337 size_t len) 338 { 339 struct virtio_vsock_sock *vvs = vsk->trans; 340 struct virtio_vsock_pkt *pkt; 341 size_t bytes, total = 0; 342 u32 free_space; 343 int err = -EFAULT; 344 345 spin_lock_bh(&vvs->rx_lock); 346 while (total < len && !list_empty(&vvs->rx_queue)) { 347 pkt = list_first_entry(&vvs->rx_queue, 348 struct virtio_vsock_pkt, list); 349 350 bytes = len - total; 351 if (bytes > pkt->len - pkt->off) 352 bytes = pkt->len - pkt->off; 353 354 /* sk_lock is held by caller so no one else can dequeue. 355 * Unlock rx_lock since memcpy_to_msg() may sleep. 356 */ 357 spin_unlock_bh(&vvs->rx_lock); 358 359 err = memcpy_to_msg(msg, pkt->buf + pkt->off, bytes); 360 if (err) 361 goto out; 362 363 spin_lock_bh(&vvs->rx_lock); 364 365 total += bytes; 366 pkt->off += bytes; 367 if (pkt->off == pkt->len) { 368 virtio_transport_dec_rx_pkt(vvs, pkt); 369 list_del(&pkt->list); 370 virtio_transport_free_pkt(pkt); 371 } 372 } 373 374 free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); 375 376 spin_unlock_bh(&vvs->rx_lock); 377 378 /* To reduce the number of credit update messages, 379 * don't update credits as long as lots of space is available. 380 * Note: the limit chosen here is arbitrary. Setting the limit 381 * too high causes extra messages. Too low causes transmitter 382 * stalls. As stalls are in theory more expensive than extra 383 * messages, we set the limit to a high value. TODO: experiment 384 * with different values. 385 */ 386 if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { 387 virtio_transport_send_credit_update(vsk, 388 VIRTIO_VSOCK_TYPE_STREAM, 389 NULL); 390 } 391 392 return total; 393 394 out: 395 if (total) 396 err = total; 397 return err; 398 } 399 400 ssize_t 401 virtio_transport_stream_dequeue(struct vsock_sock *vsk, 402 struct msghdr *msg, 403 size_t len, int flags) 404 { 405 if (flags & MSG_PEEK) 406 return virtio_transport_stream_do_peek(vsk, msg, len); 407 else 408 return virtio_transport_stream_do_dequeue(vsk, msg, len); 409 } 410 EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); 411 412 int 413 virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 414 struct msghdr *msg, 415 size_t len, int flags) 416 { 417 return -EOPNOTSUPP; 418 } 419 EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); 420 421 s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) 422 { 423 struct virtio_vsock_sock *vvs = vsk->trans; 424 s64 bytes; 425 426 spin_lock_bh(&vvs->rx_lock); 427 bytes = vvs->rx_bytes; 428 spin_unlock_bh(&vvs->rx_lock); 429 430 return bytes; 431 } 432 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); 433 434 static s64 virtio_transport_has_space(struct vsock_sock *vsk) 435 { 436 struct virtio_vsock_sock *vvs = vsk->trans; 437 s64 bytes; 438 439 bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 440 if (bytes < 0) 441 bytes = 0; 442 443 return bytes; 444 } 445 446 s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) 447 { 448 struct virtio_vsock_sock *vvs = vsk->trans; 449 s64 bytes; 450 451 spin_lock_bh(&vvs->tx_lock); 452 bytes = virtio_transport_has_space(vsk); 453 spin_unlock_bh(&vvs->tx_lock); 454 455 return bytes; 456 } 457 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); 458 459 int virtio_transport_do_socket_init(struct vsock_sock *vsk, 460 struct vsock_sock *psk) 461 { 462 struct virtio_vsock_sock *vvs; 463 464 vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); 465 if (!vvs) 466 return -ENOMEM; 467 468 vsk->trans = vvs; 469 vvs->vsk = vsk; 470 if (psk && psk->trans) { 471 struct virtio_vsock_sock *ptrans = psk->trans; 472 473 vvs->peer_buf_alloc = ptrans->peer_buf_alloc; 474 } 475 476 if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE) 477 vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE; 478 479 vvs->buf_alloc = vsk->buffer_size; 480 481 spin_lock_init(&vvs->rx_lock); 482 spin_lock_init(&vvs->tx_lock); 483 INIT_LIST_HEAD(&vvs->rx_queue); 484 485 return 0; 486 } 487 EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); 488 489 /* sk_lock held by the caller */ 490 void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val) 491 { 492 struct virtio_vsock_sock *vvs = vsk->trans; 493 494 if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE) 495 *val = VIRTIO_VSOCK_MAX_BUF_SIZE; 496 497 vvs->buf_alloc = *val; 498 499 virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, 500 NULL); 501 } 502 EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size); 503 504 int 505 virtio_transport_notify_poll_in(struct vsock_sock *vsk, 506 size_t target, 507 bool *data_ready_now) 508 { 509 if (vsock_stream_has_data(vsk)) 510 *data_ready_now = true; 511 else 512 *data_ready_now = false; 513 514 return 0; 515 } 516 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); 517 518 int 519 virtio_transport_notify_poll_out(struct vsock_sock *vsk, 520 size_t target, 521 bool *space_avail_now) 522 { 523 s64 free_space; 524 525 free_space = vsock_stream_has_space(vsk); 526 if (free_space > 0) 527 *space_avail_now = true; 528 else if (free_space == 0) 529 *space_avail_now = false; 530 531 return 0; 532 } 533 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); 534 535 int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 536 size_t target, struct vsock_transport_recv_notify_data *data) 537 { 538 return 0; 539 } 540 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); 541 542 int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 543 size_t target, struct vsock_transport_recv_notify_data *data) 544 { 545 return 0; 546 } 547 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); 548 549 int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 550 size_t target, struct vsock_transport_recv_notify_data *data) 551 { 552 return 0; 553 } 554 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); 555 556 int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 557 size_t target, ssize_t copied, bool data_read, 558 struct vsock_transport_recv_notify_data *data) 559 { 560 return 0; 561 } 562 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); 563 564 int virtio_transport_notify_send_init(struct vsock_sock *vsk, 565 struct vsock_transport_send_notify_data *data) 566 { 567 return 0; 568 } 569 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); 570 571 int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 572 struct vsock_transport_send_notify_data *data) 573 { 574 return 0; 575 } 576 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); 577 578 int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 579 struct vsock_transport_send_notify_data *data) 580 { 581 return 0; 582 } 583 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); 584 585 int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 586 ssize_t written, struct vsock_transport_send_notify_data *data) 587 { 588 return 0; 589 } 590 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); 591 592 u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) 593 { 594 return vsk->buffer_size; 595 } 596 EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); 597 598 bool virtio_transport_stream_is_active(struct vsock_sock *vsk) 599 { 600 return true; 601 } 602 EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); 603 604 bool virtio_transport_stream_allow(u32 cid, u32 port) 605 { 606 return true; 607 } 608 EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); 609 610 int virtio_transport_dgram_bind(struct vsock_sock *vsk, 611 struct sockaddr_vm *addr) 612 { 613 return -EOPNOTSUPP; 614 } 615 EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); 616 617 bool virtio_transport_dgram_allow(u32 cid, u32 port) 618 { 619 return false; 620 } 621 EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); 622 623 int virtio_transport_connect(struct vsock_sock *vsk) 624 { 625 struct virtio_vsock_pkt_info info = { 626 .op = VIRTIO_VSOCK_OP_REQUEST, 627 .type = VIRTIO_VSOCK_TYPE_STREAM, 628 .vsk = vsk, 629 }; 630 631 return virtio_transport_send_pkt_info(vsk, &info); 632 } 633 EXPORT_SYMBOL_GPL(virtio_transport_connect); 634 635 int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) 636 { 637 struct virtio_vsock_pkt_info info = { 638 .op = VIRTIO_VSOCK_OP_SHUTDOWN, 639 .type = VIRTIO_VSOCK_TYPE_STREAM, 640 .flags = (mode & RCV_SHUTDOWN ? 641 VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | 642 (mode & SEND_SHUTDOWN ? 643 VIRTIO_VSOCK_SHUTDOWN_SEND : 0), 644 .vsk = vsk, 645 }; 646 647 return virtio_transport_send_pkt_info(vsk, &info); 648 } 649 EXPORT_SYMBOL_GPL(virtio_transport_shutdown); 650 651 int 652 virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 653 struct sockaddr_vm *remote_addr, 654 struct msghdr *msg, 655 size_t dgram_len) 656 { 657 return -EOPNOTSUPP; 658 } 659 EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); 660 661 ssize_t 662 virtio_transport_stream_enqueue(struct vsock_sock *vsk, 663 struct msghdr *msg, 664 size_t len) 665 { 666 struct virtio_vsock_pkt_info info = { 667 .op = VIRTIO_VSOCK_OP_RW, 668 .type = VIRTIO_VSOCK_TYPE_STREAM, 669 .msg = msg, 670 .pkt_len = len, 671 .vsk = vsk, 672 }; 673 674 return virtio_transport_send_pkt_info(vsk, &info); 675 } 676 EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); 677 678 void virtio_transport_destruct(struct vsock_sock *vsk) 679 { 680 struct virtio_vsock_sock *vvs = vsk->trans; 681 682 kfree(vvs); 683 } 684 EXPORT_SYMBOL_GPL(virtio_transport_destruct); 685 686 static int virtio_transport_reset(struct vsock_sock *vsk, 687 struct virtio_vsock_pkt *pkt) 688 { 689 struct virtio_vsock_pkt_info info = { 690 .op = VIRTIO_VSOCK_OP_RST, 691 .type = VIRTIO_VSOCK_TYPE_STREAM, 692 .reply = !!pkt, 693 .vsk = vsk, 694 }; 695 696 /* Send RST only if the original pkt is not a RST pkt */ 697 if (pkt && le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 698 return 0; 699 700 return virtio_transport_send_pkt_info(vsk, &info); 701 } 702 703 /* Normally packets are associated with a socket. There may be no socket if an 704 * attempt was made to connect to a socket that does not exist. 705 */ 706 static int virtio_transport_reset_no_sock(const struct virtio_transport *t, 707 struct virtio_vsock_pkt *pkt) 708 { 709 struct virtio_vsock_pkt *reply; 710 struct virtio_vsock_pkt_info info = { 711 .op = VIRTIO_VSOCK_OP_RST, 712 .type = le16_to_cpu(pkt->hdr.type), 713 .reply = true, 714 }; 715 716 /* Send RST only if the original pkt is not a RST pkt */ 717 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 718 return 0; 719 720 reply = virtio_transport_alloc_pkt(&info, 0, 721 le64_to_cpu(pkt->hdr.dst_cid), 722 le32_to_cpu(pkt->hdr.dst_port), 723 le64_to_cpu(pkt->hdr.src_cid), 724 le32_to_cpu(pkt->hdr.src_port)); 725 if (!reply) 726 return -ENOMEM; 727 728 if (!t) { 729 virtio_transport_free_pkt(reply); 730 return -ENOTCONN; 731 } 732 733 return t->send_pkt(reply); 734 } 735 736 /* This function should be called with sk_lock held and SOCK_DONE set */ 737 static void virtio_transport_remove_sock(struct vsock_sock *vsk) 738 { 739 struct virtio_vsock_sock *vvs = vsk->trans; 740 struct virtio_vsock_pkt *pkt, *tmp; 741 742 /* We don't need to take rx_lock, as the socket is closing and we are 743 * removing it. 744 */ 745 list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { 746 list_del(&pkt->list); 747 virtio_transport_free_pkt(pkt); 748 } 749 750 vsock_remove_sock(vsk); 751 } 752 753 static void virtio_transport_wait_close(struct sock *sk, long timeout) 754 { 755 if (timeout) { 756 DEFINE_WAIT_FUNC(wait, woken_wake_function); 757 758 add_wait_queue(sk_sleep(sk), &wait); 759 760 do { 761 if (sk_wait_event(sk, &timeout, 762 sock_flag(sk, SOCK_DONE), &wait)) 763 break; 764 } while (!signal_pending(current) && timeout); 765 766 remove_wait_queue(sk_sleep(sk), &wait); 767 } 768 } 769 770 static void virtio_transport_do_close(struct vsock_sock *vsk, 771 bool cancel_timeout) 772 { 773 struct sock *sk = sk_vsock(vsk); 774 775 sock_set_flag(sk, SOCK_DONE); 776 vsk->peer_shutdown = SHUTDOWN_MASK; 777 if (vsock_stream_has_data(vsk) <= 0) 778 sk->sk_state = TCP_CLOSING; 779 sk->sk_state_change(sk); 780 781 if (vsk->close_work_scheduled && 782 (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { 783 vsk->close_work_scheduled = false; 784 785 virtio_transport_remove_sock(vsk); 786 787 /* Release refcnt obtained when we scheduled the timeout */ 788 sock_put(sk); 789 } 790 } 791 792 static void virtio_transport_close_timeout(struct work_struct *work) 793 { 794 struct vsock_sock *vsk = 795 container_of(work, struct vsock_sock, close_work.work); 796 struct sock *sk = sk_vsock(vsk); 797 798 sock_hold(sk); 799 lock_sock(sk); 800 801 if (!sock_flag(sk, SOCK_DONE)) { 802 (void)virtio_transport_reset(vsk, NULL); 803 804 virtio_transport_do_close(vsk, false); 805 } 806 807 vsk->close_work_scheduled = false; 808 809 release_sock(sk); 810 sock_put(sk); 811 } 812 813 /* User context, vsk->sk is locked */ 814 static bool virtio_transport_close(struct vsock_sock *vsk) 815 { 816 struct sock *sk = &vsk->sk; 817 818 if (!(sk->sk_state == TCP_ESTABLISHED || 819 sk->sk_state == TCP_CLOSING)) 820 return true; 821 822 /* Already received SHUTDOWN from peer, reply with RST */ 823 if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { 824 (void)virtio_transport_reset(vsk, NULL); 825 return true; 826 } 827 828 if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) 829 (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); 830 831 if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) 832 virtio_transport_wait_close(sk, sk->sk_lingertime); 833 834 if (sock_flag(sk, SOCK_DONE)) { 835 return true; 836 } 837 838 sock_hold(sk); 839 INIT_DELAYED_WORK(&vsk->close_work, 840 virtio_transport_close_timeout); 841 vsk->close_work_scheduled = true; 842 schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); 843 return false; 844 } 845 846 void virtio_transport_release(struct vsock_sock *vsk) 847 { 848 struct sock *sk = &vsk->sk; 849 bool remove_sock = true; 850 851 if (sk->sk_type == SOCK_STREAM) 852 remove_sock = virtio_transport_close(vsk); 853 854 if (remove_sock) { 855 sock_set_flag(sk, SOCK_DONE); 856 virtio_transport_remove_sock(vsk); 857 } 858 } 859 EXPORT_SYMBOL_GPL(virtio_transport_release); 860 861 static int 862 virtio_transport_recv_connecting(struct sock *sk, 863 struct virtio_vsock_pkt *pkt) 864 { 865 struct vsock_sock *vsk = vsock_sk(sk); 866 int err; 867 int skerr; 868 869 switch (le16_to_cpu(pkt->hdr.op)) { 870 case VIRTIO_VSOCK_OP_RESPONSE: 871 sk->sk_state = TCP_ESTABLISHED; 872 sk->sk_socket->state = SS_CONNECTED; 873 vsock_insert_connected(vsk); 874 sk->sk_state_change(sk); 875 break; 876 case VIRTIO_VSOCK_OP_INVALID: 877 break; 878 case VIRTIO_VSOCK_OP_RST: 879 skerr = ECONNRESET; 880 err = 0; 881 goto destroy; 882 default: 883 skerr = EPROTO; 884 err = -EINVAL; 885 goto destroy; 886 } 887 return 0; 888 889 destroy: 890 virtio_transport_reset(vsk, pkt); 891 sk->sk_state = TCP_CLOSE; 892 sk->sk_err = skerr; 893 sk->sk_error_report(sk); 894 return err; 895 } 896 897 static void 898 virtio_transport_recv_enqueue(struct vsock_sock *vsk, 899 struct virtio_vsock_pkt *pkt) 900 { 901 struct virtio_vsock_sock *vvs = vsk->trans; 902 bool can_enqueue, free_pkt = false; 903 904 pkt->len = le32_to_cpu(pkt->hdr.len); 905 pkt->off = 0; 906 907 spin_lock_bh(&vvs->rx_lock); 908 909 can_enqueue = virtio_transport_inc_rx_pkt(vvs, pkt); 910 if (!can_enqueue) { 911 free_pkt = true; 912 goto out; 913 } 914 915 /* Try to copy small packets into the buffer of last packet queued, 916 * to avoid wasting memory queueing the entire buffer with a small 917 * payload. 918 */ 919 if (pkt->len <= GOOD_COPY_LEN && !list_empty(&vvs->rx_queue)) { 920 struct virtio_vsock_pkt *last_pkt; 921 922 last_pkt = list_last_entry(&vvs->rx_queue, 923 struct virtio_vsock_pkt, list); 924 925 /* If there is space in the last packet queued, we copy the 926 * new packet in its buffer. 927 */ 928 if (pkt->len <= last_pkt->buf_len - last_pkt->len) { 929 memcpy(last_pkt->buf + last_pkt->len, pkt->buf, 930 pkt->len); 931 last_pkt->len += pkt->len; 932 free_pkt = true; 933 goto out; 934 } 935 } 936 937 list_add_tail(&pkt->list, &vvs->rx_queue); 938 939 out: 940 spin_unlock_bh(&vvs->rx_lock); 941 if (free_pkt) 942 virtio_transport_free_pkt(pkt); 943 } 944 945 static int 946 virtio_transport_recv_connected(struct sock *sk, 947 struct virtio_vsock_pkt *pkt) 948 { 949 struct vsock_sock *vsk = vsock_sk(sk); 950 int err = 0; 951 952 switch (le16_to_cpu(pkt->hdr.op)) { 953 case VIRTIO_VSOCK_OP_RW: 954 virtio_transport_recv_enqueue(vsk, pkt); 955 sk->sk_data_ready(sk); 956 return err; 957 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 958 sk->sk_write_space(sk); 959 break; 960 case VIRTIO_VSOCK_OP_SHUTDOWN: 961 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 962 vsk->peer_shutdown |= RCV_SHUTDOWN; 963 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 964 vsk->peer_shutdown |= SEND_SHUTDOWN; 965 if (vsk->peer_shutdown == SHUTDOWN_MASK && 966 vsock_stream_has_data(vsk) <= 0 && 967 !sock_flag(sk, SOCK_DONE)) { 968 (void)virtio_transport_reset(vsk, NULL); 969 970 virtio_transport_do_close(vsk, true); 971 } 972 if (le32_to_cpu(pkt->hdr.flags)) 973 sk->sk_state_change(sk); 974 break; 975 case VIRTIO_VSOCK_OP_RST: 976 virtio_transport_do_close(vsk, true); 977 break; 978 default: 979 err = -EINVAL; 980 break; 981 } 982 983 virtio_transport_free_pkt(pkt); 984 return err; 985 } 986 987 static void 988 virtio_transport_recv_disconnecting(struct sock *sk, 989 struct virtio_vsock_pkt *pkt) 990 { 991 struct vsock_sock *vsk = vsock_sk(sk); 992 993 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 994 virtio_transport_do_close(vsk, true); 995 } 996 997 static int 998 virtio_transport_send_response(struct vsock_sock *vsk, 999 struct virtio_vsock_pkt *pkt) 1000 { 1001 struct virtio_vsock_pkt_info info = { 1002 .op = VIRTIO_VSOCK_OP_RESPONSE, 1003 .type = VIRTIO_VSOCK_TYPE_STREAM, 1004 .remote_cid = le64_to_cpu(pkt->hdr.src_cid), 1005 .remote_port = le32_to_cpu(pkt->hdr.src_port), 1006 .reply = true, 1007 .vsk = vsk, 1008 }; 1009 1010 return virtio_transport_send_pkt_info(vsk, &info); 1011 } 1012 1013 static bool virtio_transport_space_update(struct sock *sk, 1014 struct virtio_vsock_pkt *pkt) 1015 { 1016 struct vsock_sock *vsk = vsock_sk(sk); 1017 struct virtio_vsock_sock *vvs = vsk->trans; 1018 bool space_available; 1019 1020 /* Listener sockets are not associated with any transport, so we are 1021 * not able to take the state to see if there is space available in the 1022 * remote peer, but since they are only used to receive requests, we 1023 * can assume that there is always space available in the other peer. 1024 */ 1025 if (!vvs) 1026 return true; 1027 1028 /* buf_alloc and fwd_cnt is always included in the hdr */ 1029 spin_lock_bh(&vvs->tx_lock); 1030 vvs->peer_buf_alloc = le32_to_cpu(pkt->hdr.buf_alloc); 1031 vvs->peer_fwd_cnt = le32_to_cpu(pkt->hdr.fwd_cnt); 1032 space_available = virtio_transport_has_space(vsk); 1033 spin_unlock_bh(&vvs->tx_lock); 1034 return space_available; 1035 } 1036 1037 /* Handle server socket */ 1038 static int 1039 virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt, 1040 struct virtio_transport *t) 1041 { 1042 struct vsock_sock *vsk = vsock_sk(sk); 1043 struct vsock_sock *vchild; 1044 struct sock *child; 1045 int ret; 1046 1047 if (le16_to_cpu(pkt->hdr.op) != VIRTIO_VSOCK_OP_REQUEST) { 1048 virtio_transport_reset_no_sock(t, pkt); 1049 return -EINVAL; 1050 } 1051 1052 if (sk_acceptq_is_full(sk)) { 1053 virtio_transport_reset_no_sock(t, pkt); 1054 return -ENOMEM; 1055 } 1056 1057 child = vsock_create_connected(sk); 1058 if (!child) { 1059 virtio_transport_reset_no_sock(t, pkt); 1060 return -ENOMEM; 1061 } 1062 1063 sk_acceptq_added(sk); 1064 1065 lock_sock_nested(child, SINGLE_DEPTH_NESTING); 1066 1067 child->sk_state = TCP_ESTABLISHED; 1068 1069 vchild = vsock_sk(child); 1070 vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid), 1071 le32_to_cpu(pkt->hdr.dst_port)); 1072 vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid), 1073 le32_to_cpu(pkt->hdr.src_port)); 1074 1075 ret = vsock_assign_transport(vchild, vsk); 1076 /* Transport assigned (looking at remote_addr) must be the same 1077 * where we received the request. 1078 */ 1079 if (ret || vchild->transport != &t->transport) { 1080 release_sock(child); 1081 virtio_transport_reset_no_sock(t, pkt); 1082 sock_put(child); 1083 return ret; 1084 } 1085 1086 if (virtio_transport_space_update(child, pkt)) 1087 child->sk_write_space(child); 1088 1089 vsock_insert_connected(vchild); 1090 vsock_enqueue_accept(sk, child); 1091 virtio_transport_send_response(vchild, pkt); 1092 1093 release_sock(child); 1094 1095 sk->sk_data_ready(sk); 1096 return 0; 1097 } 1098 1099 /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex 1100 * lock. 1101 */ 1102 void virtio_transport_recv_pkt(struct virtio_transport *t, 1103 struct virtio_vsock_pkt *pkt) 1104 { 1105 struct sockaddr_vm src, dst; 1106 struct vsock_sock *vsk; 1107 struct sock *sk; 1108 bool space_available; 1109 1110 vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid), 1111 le32_to_cpu(pkt->hdr.src_port)); 1112 vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid), 1113 le32_to_cpu(pkt->hdr.dst_port)); 1114 1115 trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 1116 dst.svm_cid, dst.svm_port, 1117 le32_to_cpu(pkt->hdr.len), 1118 le16_to_cpu(pkt->hdr.type), 1119 le16_to_cpu(pkt->hdr.op), 1120 le32_to_cpu(pkt->hdr.flags), 1121 le32_to_cpu(pkt->hdr.buf_alloc), 1122 le32_to_cpu(pkt->hdr.fwd_cnt)); 1123 1124 if (le16_to_cpu(pkt->hdr.type) != VIRTIO_VSOCK_TYPE_STREAM) { 1125 (void)virtio_transport_reset_no_sock(t, pkt); 1126 goto free_pkt; 1127 } 1128 1129 /* The socket must be in connected or bound table 1130 * otherwise send reset back 1131 */ 1132 sk = vsock_find_connected_socket(&src, &dst); 1133 if (!sk) { 1134 sk = vsock_find_bound_socket(&dst); 1135 if (!sk) { 1136 (void)virtio_transport_reset_no_sock(t, pkt); 1137 goto free_pkt; 1138 } 1139 } 1140 1141 vsk = vsock_sk(sk); 1142 1143 lock_sock(sk); 1144 1145 /* Check if sk has been closed before lock_sock */ 1146 if (sock_flag(sk, SOCK_DONE)) { 1147 (void)virtio_transport_reset_no_sock(t, pkt); 1148 release_sock(sk); 1149 sock_put(sk); 1150 goto free_pkt; 1151 } 1152 1153 space_available = virtio_transport_space_update(sk, pkt); 1154 1155 /* Update CID in case it has changed after a transport reset event */ 1156 vsk->local_addr.svm_cid = dst.svm_cid; 1157 1158 if (space_available) 1159 sk->sk_write_space(sk); 1160 1161 switch (sk->sk_state) { 1162 case TCP_LISTEN: 1163 virtio_transport_recv_listen(sk, pkt, t); 1164 virtio_transport_free_pkt(pkt); 1165 break; 1166 case TCP_SYN_SENT: 1167 virtio_transport_recv_connecting(sk, pkt); 1168 virtio_transport_free_pkt(pkt); 1169 break; 1170 case TCP_ESTABLISHED: 1171 virtio_transport_recv_connected(sk, pkt); 1172 break; 1173 case TCP_CLOSING: 1174 virtio_transport_recv_disconnecting(sk, pkt); 1175 virtio_transport_free_pkt(pkt); 1176 break; 1177 default: 1178 (void)virtio_transport_reset_no_sock(t, pkt); 1179 virtio_transport_free_pkt(pkt); 1180 break; 1181 } 1182 1183 release_sock(sk); 1184 1185 /* Release refcnt obtained when we fetched this socket out of the 1186 * bound or connected list. 1187 */ 1188 sock_put(sk); 1189 return; 1190 1191 free_pkt: 1192 virtio_transport_free_pkt(pkt); 1193 } 1194 EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 1195 1196 void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) 1197 { 1198 kfree(pkt->buf); 1199 kfree(pkt); 1200 } 1201 EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); 1202 1203 MODULE_LICENSE("GPL v2"); 1204 MODULE_AUTHOR("Asias He"); 1205 MODULE_DESCRIPTION("common code for virtio vsock"); 1206