1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * common code for virtio vsock 4 * 5 * Copyright (C) 2013-2015 Red Hat, Inc. 6 * Author: Asias He <asias@redhat.com> 7 * Stefan Hajnoczi <stefanha@redhat.com> 8 */ 9 #include <linux/spinlock.h> 10 #include <linux/module.h> 11 #include <linux/sched/signal.h> 12 #include <linux/ctype.h> 13 #include <linux/list.h> 14 #include <linux/virtio_vsock.h> 15 #include <uapi/linux/vsockmon.h> 16 17 #include <net/sock.h> 18 #include <net/af_vsock.h> 19 20 #define CREATE_TRACE_POINTS 21 #include <trace/events/vsock_virtio_transport_common.h> 22 23 /* How long to wait for graceful shutdown of a connection */ 24 #define VSOCK_CLOSE_TIMEOUT (8 * HZ) 25 26 /* Threshold for detecting small packets to copy */ 27 #define GOOD_COPY_LEN 128 28 29 static const struct virtio_transport * 30 virtio_transport_get_ops(struct vsock_sock *vsk) 31 { 32 const struct vsock_transport *t = vsock_core_get_transport(vsk); 33 34 if (WARN_ON(!t)) 35 return NULL; 36 37 return container_of(t, struct virtio_transport, transport); 38 } 39 40 static struct virtio_vsock_pkt * 41 virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info, 42 size_t len, 43 u32 src_cid, 44 u32 src_port, 45 u32 dst_cid, 46 u32 dst_port) 47 { 48 struct virtio_vsock_pkt *pkt; 49 int err; 50 51 pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 52 if (!pkt) 53 return NULL; 54 55 pkt->hdr.type = cpu_to_le16(info->type); 56 pkt->hdr.op = cpu_to_le16(info->op); 57 pkt->hdr.src_cid = cpu_to_le64(src_cid); 58 pkt->hdr.dst_cid = cpu_to_le64(dst_cid); 59 pkt->hdr.src_port = cpu_to_le32(src_port); 60 pkt->hdr.dst_port = cpu_to_le32(dst_port); 61 pkt->hdr.flags = cpu_to_le32(info->flags); 62 pkt->len = len; 63 pkt->hdr.len = cpu_to_le32(len); 64 pkt->reply = info->reply; 65 pkt->vsk = info->vsk; 66 67 if (info->msg && len > 0) { 68 pkt->buf = kmalloc(len, GFP_KERNEL); 69 if (!pkt->buf) 70 goto out_pkt; 71 72 pkt->buf_len = len; 73 74 err = memcpy_from_msg(pkt->buf, info->msg, len); 75 if (err) 76 goto out; 77 } 78 79 trace_virtio_transport_alloc_pkt(src_cid, src_port, 80 dst_cid, dst_port, 81 len, 82 info->type, 83 info->op, 84 info->flags); 85 86 return pkt; 87 88 out: 89 kfree(pkt->buf); 90 out_pkt: 91 kfree(pkt); 92 return NULL; 93 } 94 95 /* Packet capture */ 96 static struct sk_buff *virtio_transport_build_skb(void *opaque) 97 { 98 struct virtio_vsock_pkt *pkt = opaque; 99 struct af_vsockmon_hdr *hdr; 100 struct sk_buff *skb; 101 size_t payload_len; 102 void *payload_buf; 103 104 /* A packet could be split to fit the RX buffer, so we can retrieve 105 * the payload length from the header and the buffer pointer taking 106 * care of the offset in the original packet. 107 */ 108 payload_len = le32_to_cpu(pkt->hdr.len); 109 payload_buf = pkt->buf + pkt->off; 110 111 skb = alloc_skb(sizeof(*hdr) + sizeof(pkt->hdr) + payload_len, 112 GFP_ATOMIC); 113 if (!skb) 114 return NULL; 115 116 hdr = skb_put(skb, sizeof(*hdr)); 117 118 /* pkt->hdr is little-endian so no need to byteswap here */ 119 hdr->src_cid = pkt->hdr.src_cid; 120 hdr->src_port = pkt->hdr.src_port; 121 hdr->dst_cid = pkt->hdr.dst_cid; 122 hdr->dst_port = pkt->hdr.dst_port; 123 124 hdr->transport = cpu_to_le16(AF_VSOCK_TRANSPORT_VIRTIO); 125 hdr->len = cpu_to_le16(sizeof(pkt->hdr)); 126 memset(hdr->reserved, 0, sizeof(hdr->reserved)); 127 128 switch (le16_to_cpu(pkt->hdr.op)) { 129 case VIRTIO_VSOCK_OP_REQUEST: 130 case VIRTIO_VSOCK_OP_RESPONSE: 131 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONNECT); 132 break; 133 case VIRTIO_VSOCK_OP_RST: 134 case VIRTIO_VSOCK_OP_SHUTDOWN: 135 hdr->op = cpu_to_le16(AF_VSOCK_OP_DISCONNECT); 136 break; 137 case VIRTIO_VSOCK_OP_RW: 138 hdr->op = cpu_to_le16(AF_VSOCK_OP_PAYLOAD); 139 break; 140 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 141 case VIRTIO_VSOCK_OP_CREDIT_REQUEST: 142 hdr->op = cpu_to_le16(AF_VSOCK_OP_CONTROL); 143 break; 144 default: 145 hdr->op = cpu_to_le16(AF_VSOCK_OP_UNKNOWN); 146 break; 147 } 148 149 skb_put_data(skb, &pkt->hdr, sizeof(pkt->hdr)); 150 151 if (payload_len) { 152 skb_put_data(skb, payload_buf, payload_len); 153 } 154 155 return skb; 156 } 157 158 void virtio_transport_deliver_tap_pkt(struct virtio_vsock_pkt *pkt) 159 { 160 if (pkt->tap_delivered) 161 return; 162 163 vsock_deliver_tap(virtio_transport_build_skb, pkt); 164 pkt->tap_delivered = true; 165 } 166 EXPORT_SYMBOL_GPL(virtio_transport_deliver_tap_pkt); 167 168 /* This function can only be used on connecting/connected sockets, 169 * since a socket assigned to a transport is required. 170 * 171 * Do not use on listener sockets! 172 */ 173 static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, 174 struct virtio_vsock_pkt_info *info) 175 { 176 u32 src_cid, src_port, dst_cid, dst_port; 177 const struct virtio_transport *t_ops; 178 struct virtio_vsock_sock *vvs; 179 struct virtio_vsock_pkt *pkt; 180 u32 pkt_len = info->pkt_len; 181 182 t_ops = virtio_transport_get_ops(vsk); 183 if (unlikely(!t_ops)) 184 return -EFAULT; 185 186 src_cid = t_ops->transport.get_local_cid(); 187 src_port = vsk->local_addr.svm_port; 188 if (!info->remote_cid) { 189 dst_cid = vsk->remote_addr.svm_cid; 190 dst_port = vsk->remote_addr.svm_port; 191 } else { 192 dst_cid = info->remote_cid; 193 dst_port = info->remote_port; 194 } 195 196 vvs = vsk->trans; 197 198 /* we can send less than pkt_len bytes */ 199 if (pkt_len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) 200 pkt_len = VIRTIO_VSOCK_MAX_PKT_BUF_SIZE; 201 202 /* virtio_transport_get_credit might return less than pkt_len credit */ 203 pkt_len = virtio_transport_get_credit(vvs, pkt_len); 204 205 /* Do not send zero length OP_RW pkt */ 206 if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 207 return pkt_len; 208 209 pkt = virtio_transport_alloc_pkt(info, pkt_len, 210 src_cid, src_port, 211 dst_cid, dst_port); 212 if (!pkt) { 213 virtio_transport_put_credit(vvs, pkt_len); 214 return -ENOMEM; 215 } 216 217 virtio_transport_inc_tx_pkt(vvs, pkt); 218 219 return t_ops->send_pkt(pkt); 220 } 221 222 static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 223 struct virtio_vsock_pkt *pkt) 224 { 225 if (vvs->rx_bytes + pkt->len > vvs->buf_alloc) 226 return false; 227 228 vvs->rx_bytes += pkt->len; 229 return true; 230 } 231 232 static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 233 struct virtio_vsock_pkt *pkt) 234 { 235 vvs->rx_bytes -= pkt->len; 236 vvs->fwd_cnt += pkt->len; 237 } 238 239 void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) 240 { 241 spin_lock_bh(&vvs->rx_lock); 242 vvs->last_fwd_cnt = vvs->fwd_cnt; 243 pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 244 pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); 245 spin_unlock_bh(&vvs->rx_lock); 246 } 247 EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); 248 249 u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) 250 { 251 u32 ret; 252 253 spin_lock_bh(&vvs->tx_lock); 254 ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 255 if (ret > credit) 256 ret = credit; 257 vvs->tx_cnt += ret; 258 spin_unlock_bh(&vvs->tx_lock); 259 260 return ret; 261 } 262 EXPORT_SYMBOL_GPL(virtio_transport_get_credit); 263 264 void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) 265 { 266 spin_lock_bh(&vvs->tx_lock); 267 vvs->tx_cnt -= credit; 268 spin_unlock_bh(&vvs->tx_lock); 269 } 270 EXPORT_SYMBOL_GPL(virtio_transport_put_credit); 271 272 static int virtio_transport_send_credit_update(struct vsock_sock *vsk, 273 int type, 274 struct virtio_vsock_hdr *hdr) 275 { 276 struct virtio_vsock_pkt_info info = { 277 .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, 278 .type = type, 279 .vsk = vsk, 280 }; 281 282 return virtio_transport_send_pkt_info(vsk, &info); 283 } 284 285 static ssize_t 286 virtio_transport_stream_do_peek(struct vsock_sock *vsk, 287 struct msghdr *msg, 288 size_t len) 289 { 290 struct virtio_vsock_sock *vvs = vsk->trans; 291 struct virtio_vsock_pkt *pkt; 292 size_t bytes, total = 0, off; 293 int err = -EFAULT; 294 295 spin_lock_bh(&vvs->rx_lock); 296 297 list_for_each_entry(pkt, &vvs->rx_queue, list) { 298 off = pkt->off; 299 300 if (total == len) 301 break; 302 303 while (total < len && off < pkt->len) { 304 bytes = len - total; 305 if (bytes > pkt->len - off) 306 bytes = pkt->len - off; 307 308 /* sk_lock is held by caller so no one else can dequeue. 309 * Unlock rx_lock since memcpy_to_msg() may sleep. 310 */ 311 spin_unlock_bh(&vvs->rx_lock); 312 313 err = memcpy_to_msg(msg, pkt->buf + off, bytes); 314 if (err) 315 goto out; 316 317 spin_lock_bh(&vvs->rx_lock); 318 319 total += bytes; 320 off += bytes; 321 } 322 } 323 324 spin_unlock_bh(&vvs->rx_lock); 325 326 return total; 327 328 out: 329 if (total) 330 err = total; 331 return err; 332 } 333 334 static ssize_t 335 virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, 336 struct msghdr *msg, 337 size_t len) 338 { 339 struct virtio_vsock_sock *vvs = vsk->trans; 340 struct virtio_vsock_pkt *pkt; 341 size_t bytes, total = 0; 342 u32 free_space; 343 int err = -EFAULT; 344 345 spin_lock_bh(&vvs->rx_lock); 346 while (total < len && !list_empty(&vvs->rx_queue)) { 347 pkt = list_first_entry(&vvs->rx_queue, 348 struct virtio_vsock_pkt, list); 349 350 bytes = len - total; 351 if (bytes > pkt->len - pkt->off) 352 bytes = pkt->len - pkt->off; 353 354 /* sk_lock is held by caller so no one else can dequeue. 355 * Unlock rx_lock since memcpy_to_msg() may sleep. 356 */ 357 spin_unlock_bh(&vvs->rx_lock); 358 359 err = memcpy_to_msg(msg, pkt->buf + pkt->off, bytes); 360 if (err) 361 goto out; 362 363 spin_lock_bh(&vvs->rx_lock); 364 365 total += bytes; 366 pkt->off += bytes; 367 if (pkt->off == pkt->len) { 368 virtio_transport_dec_rx_pkt(vvs, pkt); 369 list_del(&pkt->list); 370 virtio_transport_free_pkt(pkt); 371 } 372 } 373 374 free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt); 375 376 spin_unlock_bh(&vvs->rx_lock); 377 378 /* To reduce the number of credit update messages, 379 * don't update credits as long as lots of space is available. 380 * Note: the limit chosen here is arbitrary. Setting the limit 381 * too high causes extra messages. Too low causes transmitter 382 * stalls. As stalls are in theory more expensive than extra 383 * messages, we set the limit to a high value. TODO: experiment 384 * with different values. 385 */ 386 if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { 387 virtio_transport_send_credit_update(vsk, 388 VIRTIO_VSOCK_TYPE_STREAM, 389 NULL); 390 } 391 392 return total; 393 394 out: 395 if (total) 396 err = total; 397 return err; 398 } 399 400 ssize_t 401 virtio_transport_stream_dequeue(struct vsock_sock *vsk, 402 struct msghdr *msg, 403 size_t len, int flags) 404 { 405 if (flags & MSG_PEEK) 406 return virtio_transport_stream_do_peek(vsk, msg, len); 407 else 408 return virtio_transport_stream_do_dequeue(vsk, msg, len); 409 } 410 EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); 411 412 int 413 virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 414 struct msghdr *msg, 415 size_t len, int flags) 416 { 417 return -EOPNOTSUPP; 418 } 419 EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); 420 421 s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) 422 { 423 struct virtio_vsock_sock *vvs = vsk->trans; 424 s64 bytes; 425 426 spin_lock_bh(&vvs->rx_lock); 427 bytes = vvs->rx_bytes; 428 spin_unlock_bh(&vvs->rx_lock); 429 430 return bytes; 431 } 432 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); 433 434 static s64 virtio_transport_has_space(struct vsock_sock *vsk) 435 { 436 struct virtio_vsock_sock *vvs = vsk->trans; 437 s64 bytes; 438 439 bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 440 if (bytes < 0) 441 bytes = 0; 442 443 return bytes; 444 } 445 446 s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) 447 { 448 struct virtio_vsock_sock *vvs = vsk->trans; 449 s64 bytes; 450 451 spin_lock_bh(&vvs->tx_lock); 452 bytes = virtio_transport_has_space(vsk); 453 spin_unlock_bh(&vvs->tx_lock); 454 455 return bytes; 456 } 457 EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); 458 459 int virtio_transport_do_socket_init(struct vsock_sock *vsk, 460 struct vsock_sock *psk) 461 { 462 struct virtio_vsock_sock *vvs; 463 464 vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); 465 if (!vvs) 466 return -ENOMEM; 467 468 vsk->trans = vvs; 469 vvs->vsk = vsk; 470 if (psk && psk->trans) { 471 struct virtio_vsock_sock *ptrans = psk->trans; 472 473 vvs->peer_buf_alloc = ptrans->peer_buf_alloc; 474 } 475 476 if (vsk->buffer_size > VIRTIO_VSOCK_MAX_BUF_SIZE) 477 vsk->buffer_size = VIRTIO_VSOCK_MAX_BUF_SIZE; 478 479 vvs->buf_alloc = vsk->buffer_size; 480 481 spin_lock_init(&vvs->rx_lock); 482 spin_lock_init(&vvs->tx_lock); 483 INIT_LIST_HEAD(&vvs->rx_queue); 484 485 return 0; 486 } 487 EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); 488 489 /* sk_lock held by the caller */ 490 void virtio_transport_notify_buffer_size(struct vsock_sock *vsk, u64 *val) 491 { 492 struct virtio_vsock_sock *vvs = vsk->trans; 493 494 if (*val > VIRTIO_VSOCK_MAX_BUF_SIZE) 495 *val = VIRTIO_VSOCK_MAX_BUF_SIZE; 496 497 vvs->buf_alloc = *val; 498 499 virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, 500 NULL); 501 } 502 EXPORT_SYMBOL_GPL(virtio_transport_notify_buffer_size); 503 504 int 505 virtio_transport_notify_poll_in(struct vsock_sock *vsk, 506 size_t target, 507 bool *data_ready_now) 508 { 509 if (vsock_stream_has_data(vsk)) 510 *data_ready_now = true; 511 else 512 *data_ready_now = false; 513 514 return 0; 515 } 516 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); 517 518 int 519 virtio_transport_notify_poll_out(struct vsock_sock *vsk, 520 size_t target, 521 bool *space_avail_now) 522 { 523 s64 free_space; 524 525 free_space = vsock_stream_has_space(vsk); 526 if (free_space > 0) 527 *space_avail_now = true; 528 else if (free_space == 0) 529 *space_avail_now = false; 530 531 return 0; 532 } 533 EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); 534 535 int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 536 size_t target, struct vsock_transport_recv_notify_data *data) 537 { 538 return 0; 539 } 540 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); 541 542 int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 543 size_t target, struct vsock_transport_recv_notify_data *data) 544 { 545 return 0; 546 } 547 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); 548 549 int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 550 size_t target, struct vsock_transport_recv_notify_data *data) 551 { 552 return 0; 553 } 554 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); 555 556 int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 557 size_t target, ssize_t copied, bool data_read, 558 struct vsock_transport_recv_notify_data *data) 559 { 560 return 0; 561 } 562 EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); 563 564 int virtio_transport_notify_send_init(struct vsock_sock *vsk, 565 struct vsock_transport_send_notify_data *data) 566 { 567 return 0; 568 } 569 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); 570 571 int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 572 struct vsock_transport_send_notify_data *data) 573 { 574 return 0; 575 } 576 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); 577 578 int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 579 struct vsock_transport_send_notify_data *data) 580 { 581 return 0; 582 } 583 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); 584 585 int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 586 ssize_t written, struct vsock_transport_send_notify_data *data) 587 { 588 return 0; 589 } 590 EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); 591 592 u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) 593 { 594 return vsk->buffer_size; 595 } 596 EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); 597 598 bool virtio_transport_stream_is_active(struct vsock_sock *vsk) 599 { 600 return true; 601 } 602 EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); 603 604 bool virtio_transport_stream_allow(u32 cid, u32 port) 605 { 606 return true; 607 } 608 EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); 609 610 int virtio_transport_dgram_bind(struct vsock_sock *vsk, 611 struct sockaddr_vm *addr) 612 { 613 return -EOPNOTSUPP; 614 } 615 EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); 616 617 bool virtio_transport_dgram_allow(u32 cid, u32 port) 618 { 619 return false; 620 } 621 EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); 622 623 int virtio_transport_connect(struct vsock_sock *vsk) 624 { 625 struct virtio_vsock_pkt_info info = { 626 .op = VIRTIO_VSOCK_OP_REQUEST, 627 .type = VIRTIO_VSOCK_TYPE_STREAM, 628 .vsk = vsk, 629 }; 630 631 return virtio_transport_send_pkt_info(vsk, &info); 632 } 633 EXPORT_SYMBOL_GPL(virtio_transport_connect); 634 635 int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) 636 { 637 struct virtio_vsock_pkt_info info = { 638 .op = VIRTIO_VSOCK_OP_SHUTDOWN, 639 .type = VIRTIO_VSOCK_TYPE_STREAM, 640 .flags = (mode & RCV_SHUTDOWN ? 641 VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | 642 (mode & SEND_SHUTDOWN ? 643 VIRTIO_VSOCK_SHUTDOWN_SEND : 0), 644 .vsk = vsk, 645 }; 646 647 return virtio_transport_send_pkt_info(vsk, &info); 648 } 649 EXPORT_SYMBOL_GPL(virtio_transport_shutdown); 650 651 int 652 virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 653 struct sockaddr_vm *remote_addr, 654 struct msghdr *msg, 655 size_t dgram_len) 656 { 657 return -EOPNOTSUPP; 658 } 659 EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); 660 661 ssize_t 662 virtio_transport_stream_enqueue(struct vsock_sock *vsk, 663 struct msghdr *msg, 664 size_t len) 665 { 666 struct virtio_vsock_pkt_info info = { 667 .op = VIRTIO_VSOCK_OP_RW, 668 .type = VIRTIO_VSOCK_TYPE_STREAM, 669 .msg = msg, 670 .pkt_len = len, 671 .vsk = vsk, 672 }; 673 674 return virtio_transport_send_pkt_info(vsk, &info); 675 } 676 EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); 677 678 void virtio_transport_destruct(struct vsock_sock *vsk) 679 { 680 struct virtio_vsock_sock *vvs = vsk->trans; 681 682 kfree(vvs); 683 } 684 EXPORT_SYMBOL_GPL(virtio_transport_destruct); 685 686 static int virtio_transport_reset(struct vsock_sock *vsk, 687 struct virtio_vsock_pkt *pkt) 688 { 689 struct virtio_vsock_pkt_info info = { 690 .op = VIRTIO_VSOCK_OP_RST, 691 .type = VIRTIO_VSOCK_TYPE_STREAM, 692 .reply = !!pkt, 693 .vsk = vsk, 694 }; 695 696 /* Send RST only if the original pkt is not a RST pkt */ 697 if (pkt && le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 698 return 0; 699 700 return virtio_transport_send_pkt_info(vsk, &info); 701 } 702 703 /* Normally packets are associated with a socket. There may be no socket if an 704 * attempt was made to connect to a socket that does not exist. 705 */ 706 static int virtio_transport_reset_no_sock(const struct virtio_transport *t, 707 struct virtio_vsock_pkt *pkt) 708 { 709 struct virtio_vsock_pkt *reply; 710 struct virtio_vsock_pkt_info info = { 711 .op = VIRTIO_VSOCK_OP_RST, 712 .type = le16_to_cpu(pkt->hdr.type), 713 .reply = true, 714 }; 715 716 /* Send RST only if the original pkt is not a RST pkt */ 717 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 718 return 0; 719 720 reply = virtio_transport_alloc_pkt(&info, 0, 721 le64_to_cpu(pkt->hdr.dst_cid), 722 le32_to_cpu(pkt->hdr.dst_port), 723 le64_to_cpu(pkt->hdr.src_cid), 724 le32_to_cpu(pkt->hdr.src_port)); 725 if (!reply) 726 return -ENOMEM; 727 728 if (!t) { 729 virtio_transport_free_pkt(reply); 730 return -ENOTCONN; 731 } 732 733 return t->send_pkt(reply); 734 } 735 736 static void virtio_transport_wait_close(struct sock *sk, long timeout) 737 { 738 if (timeout) { 739 DEFINE_WAIT_FUNC(wait, woken_wake_function); 740 741 add_wait_queue(sk_sleep(sk), &wait); 742 743 do { 744 if (sk_wait_event(sk, &timeout, 745 sock_flag(sk, SOCK_DONE), &wait)) 746 break; 747 } while (!signal_pending(current) && timeout); 748 749 remove_wait_queue(sk_sleep(sk), &wait); 750 } 751 } 752 753 static void virtio_transport_do_close(struct vsock_sock *vsk, 754 bool cancel_timeout) 755 { 756 struct sock *sk = sk_vsock(vsk); 757 758 sock_set_flag(sk, SOCK_DONE); 759 vsk->peer_shutdown = SHUTDOWN_MASK; 760 if (vsock_stream_has_data(vsk) <= 0) 761 sk->sk_state = TCP_CLOSING; 762 sk->sk_state_change(sk); 763 764 if (vsk->close_work_scheduled && 765 (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { 766 vsk->close_work_scheduled = false; 767 768 vsock_remove_sock(vsk); 769 770 /* Release refcnt obtained when we scheduled the timeout */ 771 sock_put(sk); 772 } 773 } 774 775 static void virtio_transport_close_timeout(struct work_struct *work) 776 { 777 struct vsock_sock *vsk = 778 container_of(work, struct vsock_sock, close_work.work); 779 struct sock *sk = sk_vsock(vsk); 780 781 sock_hold(sk); 782 lock_sock(sk); 783 784 if (!sock_flag(sk, SOCK_DONE)) { 785 (void)virtio_transport_reset(vsk, NULL); 786 787 virtio_transport_do_close(vsk, false); 788 } 789 790 vsk->close_work_scheduled = false; 791 792 release_sock(sk); 793 sock_put(sk); 794 } 795 796 /* User context, vsk->sk is locked */ 797 static bool virtio_transport_close(struct vsock_sock *vsk) 798 { 799 struct sock *sk = &vsk->sk; 800 801 if (!(sk->sk_state == TCP_ESTABLISHED || 802 sk->sk_state == TCP_CLOSING)) 803 return true; 804 805 /* Already received SHUTDOWN from peer, reply with RST */ 806 if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { 807 (void)virtio_transport_reset(vsk, NULL); 808 return true; 809 } 810 811 if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) 812 (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); 813 814 if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) 815 virtio_transport_wait_close(sk, sk->sk_lingertime); 816 817 if (sock_flag(sk, SOCK_DONE)) { 818 return true; 819 } 820 821 sock_hold(sk); 822 INIT_DELAYED_WORK(&vsk->close_work, 823 virtio_transport_close_timeout); 824 vsk->close_work_scheduled = true; 825 schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); 826 return false; 827 } 828 829 void virtio_transport_release(struct vsock_sock *vsk) 830 { 831 struct virtio_vsock_sock *vvs = vsk->trans; 832 struct virtio_vsock_pkt *pkt, *tmp; 833 struct sock *sk = &vsk->sk; 834 bool remove_sock = true; 835 836 if (sk->sk_type == SOCK_STREAM) 837 remove_sock = virtio_transport_close(vsk); 838 839 list_for_each_entry_safe(pkt, tmp, &vvs->rx_queue, list) { 840 list_del(&pkt->list); 841 virtio_transport_free_pkt(pkt); 842 } 843 844 if (remove_sock) 845 vsock_remove_sock(vsk); 846 } 847 EXPORT_SYMBOL_GPL(virtio_transport_release); 848 849 static int 850 virtio_transport_recv_connecting(struct sock *sk, 851 struct virtio_vsock_pkt *pkt) 852 { 853 struct vsock_sock *vsk = vsock_sk(sk); 854 int err; 855 int skerr; 856 857 switch (le16_to_cpu(pkt->hdr.op)) { 858 case VIRTIO_VSOCK_OP_RESPONSE: 859 sk->sk_state = TCP_ESTABLISHED; 860 sk->sk_socket->state = SS_CONNECTED; 861 vsock_insert_connected(vsk); 862 sk->sk_state_change(sk); 863 break; 864 case VIRTIO_VSOCK_OP_INVALID: 865 break; 866 case VIRTIO_VSOCK_OP_RST: 867 skerr = ECONNRESET; 868 err = 0; 869 goto destroy; 870 default: 871 skerr = EPROTO; 872 err = -EINVAL; 873 goto destroy; 874 } 875 return 0; 876 877 destroy: 878 virtio_transport_reset(vsk, pkt); 879 sk->sk_state = TCP_CLOSE; 880 sk->sk_err = skerr; 881 sk->sk_error_report(sk); 882 return err; 883 } 884 885 static void 886 virtio_transport_recv_enqueue(struct vsock_sock *vsk, 887 struct virtio_vsock_pkt *pkt) 888 { 889 struct virtio_vsock_sock *vvs = vsk->trans; 890 bool can_enqueue, free_pkt = false; 891 892 pkt->len = le32_to_cpu(pkt->hdr.len); 893 pkt->off = 0; 894 895 spin_lock_bh(&vvs->rx_lock); 896 897 can_enqueue = virtio_transport_inc_rx_pkt(vvs, pkt); 898 if (!can_enqueue) { 899 free_pkt = true; 900 goto out; 901 } 902 903 /* Try to copy small packets into the buffer of last packet queued, 904 * to avoid wasting memory queueing the entire buffer with a small 905 * payload. 906 */ 907 if (pkt->len <= GOOD_COPY_LEN && !list_empty(&vvs->rx_queue)) { 908 struct virtio_vsock_pkt *last_pkt; 909 910 last_pkt = list_last_entry(&vvs->rx_queue, 911 struct virtio_vsock_pkt, list); 912 913 /* If there is space in the last packet queued, we copy the 914 * new packet in its buffer. 915 */ 916 if (pkt->len <= last_pkt->buf_len - last_pkt->len) { 917 memcpy(last_pkt->buf + last_pkt->len, pkt->buf, 918 pkt->len); 919 last_pkt->len += pkt->len; 920 free_pkt = true; 921 goto out; 922 } 923 } 924 925 list_add_tail(&pkt->list, &vvs->rx_queue); 926 927 out: 928 spin_unlock_bh(&vvs->rx_lock); 929 if (free_pkt) 930 virtio_transport_free_pkt(pkt); 931 } 932 933 static int 934 virtio_transport_recv_connected(struct sock *sk, 935 struct virtio_vsock_pkt *pkt) 936 { 937 struct vsock_sock *vsk = vsock_sk(sk); 938 int err = 0; 939 940 switch (le16_to_cpu(pkt->hdr.op)) { 941 case VIRTIO_VSOCK_OP_RW: 942 virtio_transport_recv_enqueue(vsk, pkt); 943 sk->sk_data_ready(sk); 944 return err; 945 case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 946 sk->sk_write_space(sk); 947 break; 948 case VIRTIO_VSOCK_OP_SHUTDOWN: 949 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 950 vsk->peer_shutdown |= RCV_SHUTDOWN; 951 if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 952 vsk->peer_shutdown |= SEND_SHUTDOWN; 953 if (vsk->peer_shutdown == SHUTDOWN_MASK && 954 vsock_stream_has_data(vsk) <= 0 && 955 !sock_flag(sk, SOCK_DONE)) { 956 (void)virtio_transport_reset(vsk, NULL); 957 958 virtio_transport_do_close(vsk, true); 959 } 960 if (le32_to_cpu(pkt->hdr.flags)) 961 sk->sk_state_change(sk); 962 break; 963 case VIRTIO_VSOCK_OP_RST: 964 virtio_transport_do_close(vsk, true); 965 break; 966 default: 967 err = -EINVAL; 968 break; 969 } 970 971 virtio_transport_free_pkt(pkt); 972 return err; 973 } 974 975 static void 976 virtio_transport_recv_disconnecting(struct sock *sk, 977 struct virtio_vsock_pkt *pkt) 978 { 979 struct vsock_sock *vsk = vsock_sk(sk); 980 981 if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 982 virtio_transport_do_close(vsk, true); 983 } 984 985 static int 986 virtio_transport_send_response(struct vsock_sock *vsk, 987 struct virtio_vsock_pkt *pkt) 988 { 989 struct virtio_vsock_pkt_info info = { 990 .op = VIRTIO_VSOCK_OP_RESPONSE, 991 .type = VIRTIO_VSOCK_TYPE_STREAM, 992 .remote_cid = le64_to_cpu(pkt->hdr.src_cid), 993 .remote_port = le32_to_cpu(pkt->hdr.src_port), 994 .reply = true, 995 .vsk = vsk, 996 }; 997 998 return virtio_transport_send_pkt_info(vsk, &info); 999 } 1000 1001 static bool virtio_transport_space_update(struct sock *sk, 1002 struct virtio_vsock_pkt *pkt) 1003 { 1004 struct vsock_sock *vsk = vsock_sk(sk); 1005 struct virtio_vsock_sock *vvs = vsk->trans; 1006 bool space_available; 1007 1008 /* Listener sockets are not associated with any transport, so we are 1009 * not able to take the state to see if there is space available in the 1010 * remote peer, but since they are only used to receive requests, we 1011 * can assume that there is always space available in the other peer. 1012 */ 1013 if (!vvs) 1014 return true; 1015 1016 /* buf_alloc and fwd_cnt is always included in the hdr */ 1017 spin_lock_bh(&vvs->tx_lock); 1018 vvs->peer_buf_alloc = le32_to_cpu(pkt->hdr.buf_alloc); 1019 vvs->peer_fwd_cnt = le32_to_cpu(pkt->hdr.fwd_cnt); 1020 space_available = virtio_transport_has_space(vsk); 1021 spin_unlock_bh(&vvs->tx_lock); 1022 return space_available; 1023 } 1024 1025 /* Handle server socket */ 1026 static int 1027 virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt, 1028 struct virtio_transport *t) 1029 { 1030 struct vsock_sock *vsk = vsock_sk(sk); 1031 struct vsock_sock *vchild; 1032 struct sock *child; 1033 int ret; 1034 1035 if (le16_to_cpu(pkt->hdr.op) != VIRTIO_VSOCK_OP_REQUEST) { 1036 virtio_transport_reset_no_sock(t, pkt); 1037 return -EINVAL; 1038 } 1039 1040 if (sk_acceptq_is_full(sk)) { 1041 virtio_transport_reset_no_sock(t, pkt); 1042 return -ENOMEM; 1043 } 1044 1045 child = vsock_create_connected(sk); 1046 if (!child) { 1047 virtio_transport_reset_no_sock(t, pkt); 1048 return -ENOMEM; 1049 } 1050 1051 sk_acceptq_added(sk); 1052 1053 lock_sock_nested(child, SINGLE_DEPTH_NESTING); 1054 1055 child->sk_state = TCP_ESTABLISHED; 1056 1057 vchild = vsock_sk(child); 1058 vsock_addr_init(&vchild->local_addr, le64_to_cpu(pkt->hdr.dst_cid), 1059 le32_to_cpu(pkt->hdr.dst_port)); 1060 vsock_addr_init(&vchild->remote_addr, le64_to_cpu(pkt->hdr.src_cid), 1061 le32_to_cpu(pkt->hdr.src_port)); 1062 1063 ret = vsock_assign_transport(vchild, vsk); 1064 /* Transport assigned (looking at remote_addr) must be the same 1065 * where we received the request. 1066 */ 1067 if (ret || vchild->transport != &t->transport) { 1068 release_sock(child); 1069 virtio_transport_reset_no_sock(t, pkt); 1070 sock_put(child); 1071 return ret; 1072 } 1073 1074 if (virtio_transport_space_update(child, pkt)) 1075 child->sk_write_space(child); 1076 1077 vsock_insert_connected(vchild); 1078 vsock_enqueue_accept(sk, child); 1079 virtio_transport_send_response(vchild, pkt); 1080 1081 release_sock(child); 1082 1083 sk->sk_data_ready(sk); 1084 return 0; 1085 } 1086 1087 /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex 1088 * lock. 1089 */ 1090 void virtio_transport_recv_pkt(struct virtio_transport *t, 1091 struct virtio_vsock_pkt *pkt) 1092 { 1093 struct sockaddr_vm src, dst; 1094 struct vsock_sock *vsk; 1095 struct sock *sk; 1096 bool space_available; 1097 1098 vsock_addr_init(&src, le64_to_cpu(pkt->hdr.src_cid), 1099 le32_to_cpu(pkt->hdr.src_port)); 1100 vsock_addr_init(&dst, le64_to_cpu(pkt->hdr.dst_cid), 1101 le32_to_cpu(pkt->hdr.dst_port)); 1102 1103 trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 1104 dst.svm_cid, dst.svm_port, 1105 le32_to_cpu(pkt->hdr.len), 1106 le16_to_cpu(pkt->hdr.type), 1107 le16_to_cpu(pkt->hdr.op), 1108 le32_to_cpu(pkt->hdr.flags), 1109 le32_to_cpu(pkt->hdr.buf_alloc), 1110 le32_to_cpu(pkt->hdr.fwd_cnt)); 1111 1112 if (le16_to_cpu(pkt->hdr.type) != VIRTIO_VSOCK_TYPE_STREAM) { 1113 (void)virtio_transport_reset_no_sock(t, pkt); 1114 goto free_pkt; 1115 } 1116 1117 /* The socket must be in connected or bound table 1118 * otherwise send reset back 1119 */ 1120 sk = vsock_find_connected_socket(&src, &dst); 1121 if (!sk) { 1122 sk = vsock_find_bound_socket(&dst); 1123 if (!sk) { 1124 (void)virtio_transport_reset_no_sock(t, pkt); 1125 goto free_pkt; 1126 } 1127 } 1128 1129 vsk = vsock_sk(sk); 1130 1131 space_available = virtio_transport_space_update(sk, pkt); 1132 1133 lock_sock(sk); 1134 1135 /* Update CID in case it has changed after a transport reset event */ 1136 vsk->local_addr.svm_cid = dst.svm_cid; 1137 1138 if (space_available) 1139 sk->sk_write_space(sk); 1140 1141 switch (sk->sk_state) { 1142 case TCP_LISTEN: 1143 virtio_transport_recv_listen(sk, pkt, t); 1144 virtio_transport_free_pkt(pkt); 1145 break; 1146 case TCP_SYN_SENT: 1147 virtio_transport_recv_connecting(sk, pkt); 1148 virtio_transport_free_pkt(pkt); 1149 break; 1150 case TCP_ESTABLISHED: 1151 virtio_transport_recv_connected(sk, pkt); 1152 break; 1153 case TCP_CLOSING: 1154 virtio_transport_recv_disconnecting(sk, pkt); 1155 virtio_transport_free_pkt(pkt); 1156 break; 1157 default: 1158 (void)virtio_transport_reset_no_sock(t, pkt); 1159 virtio_transport_free_pkt(pkt); 1160 break; 1161 } 1162 1163 release_sock(sk); 1164 1165 /* Release refcnt obtained when we fetched this socket out of the 1166 * bound or connected list. 1167 */ 1168 sock_put(sk); 1169 return; 1170 1171 free_pkt: 1172 virtio_transport_free_pkt(pkt); 1173 } 1174 EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 1175 1176 void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) 1177 { 1178 kfree(pkt->buf); 1179 kfree(pkt); 1180 } 1181 EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); 1182 1183 MODULE_LICENSE("GPL v2"); 1184 MODULE_AUTHOR("Asias He"); 1185 MODULE_DESCRIPTION("common code for virtio vsock"); 1186