xref: /openbmc/linux/net/vmw_vsock/af_vsock.c (revision 9726bfcd)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VMware vSockets Driver
4  *
5  * Copyright (C) 2007-2013 VMware, Inc. All rights reserved.
6  */
7 
8 /* Implementation notes:
9  *
10  * - There are two kinds of sockets: those created by user action (such as
11  * calling socket(2)) and those created by incoming connection request packets.
12  *
13  * - There are two "global" tables, one for bound sockets (sockets that have
14  * specified an address that they are responsible for) and one for connected
15  * sockets (sockets that have established a connection with another socket).
16  * These tables are "global" in that all sockets on the system are placed
17  * within them. - Note, though, that the bound table contains an extra entry
18  * for a list of unbound sockets and SOCK_DGRAM sockets will always remain in
19  * that list. The bound table is used solely for lookup of sockets when packets
20  * are received and that's not necessary for SOCK_DGRAM sockets since we create
21  * a datagram handle for each and need not perform a lookup.  Keeping SOCK_DGRAM
22  * sockets out of the bound hash buckets will reduce the chance of collisions
23  * when looking for SOCK_STREAM sockets and prevents us from having to check the
24  * socket type in the hash table lookups.
25  *
26  * - Sockets created by user action will either be "client" sockets that
27  * initiate a connection or "server" sockets that listen for connections; we do
28  * not support simultaneous connects (two "client" sockets connecting).
29  *
30  * - "Server" sockets are referred to as listener sockets throughout this
31  * implementation because they are in the TCP_LISTEN state.  When a
32  * connection request is received (the second kind of socket mentioned above),
33  * we create a new socket and refer to it as a pending socket.  These pending
34  * sockets are placed on the pending connection list of the listener socket.
35  * When future packets are received for the address the listener socket is
36  * bound to, we check if the source of the packet is from one that has an
37  * existing pending connection.  If it does, we process the packet for the
38  * pending socket.  When that socket reaches the connected state, it is removed
39  * from the listener socket's pending list and enqueued in the listener
40  * socket's accept queue.  Callers of accept(2) will accept connected sockets
41  * from the listener socket's accept queue.  If the socket cannot be accepted
42  * for some reason then it is marked rejected.  Once the connection is
43  * accepted, it is owned by the user process and the responsibility for cleanup
44  * falls with that user process.
45  *
46  * - It is possible that these pending sockets will never reach the connected
47  * state; in fact, we may never receive another packet after the connection
48  * request.  Because of this, we must schedule a cleanup function to run in the
49  * future, after some amount of time passes where a connection should have been
50  * established.  This function ensures that the socket is off all lists so it
51  * cannot be retrieved, then drops all references to the socket so it is cleaned
52  * up (sock_put() -> sk_free() -> our sk_destruct implementation).  Note this
53  * function will also cleanup rejected sockets, those that reach the connected
54  * state but leave it before they have been accepted.
55  *
56  * - Lock ordering for pending or accept queue sockets is:
57  *
58  *     lock_sock(listener);
59  *     lock_sock_nested(pending, SINGLE_DEPTH_NESTING);
60  *
61  * Using explicit nested locking keeps lockdep happy since normally only one
62  * lock of a given class may be taken at a time.
63  *
64  * - Sockets created by user action will be cleaned up when the user process
65  * calls close(2), causing our release implementation to be called. Our release
66  * implementation will perform some cleanup then drop the last reference so our
67  * sk_destruct implementation is invoked.  Our sk_destruct implementation will
68  * perform additional cleanup that's common for both types of sockets.
69  *
70  * - A socket's reference count is what ensures that the structure won't be
71  * freed.  Each entry in a list (such as the "global" bound and connected tables
72  * and the listener socket's pending list and connected queue) ensures a
73  * reference.  When we defer work until process context and pass a socket as our
74  * argument, we must ensure the reference count is increased to ensure the
75  * socket isn't freed before the function is run; the deferred function will
76  * then drop the reference.
77  *
78  * - sk->sk_state uses the TCP state constants because they are widely used by
79  * other address families and exposed to userspace tools like ss(8):
80  *
81  *   TCP_CLOSE - unconnected
82  *   TCP_SYN_SENT - connecting
83  *   TCP_ESTABLISHED - connected
84  *   TCP_CLOSING - disconnecting
85  *   TCP_LISTEN - listening
86  */
87 
88 #include <linux/types.h>
89 #include <linux/bitops.h>
90 #include <linux/cred.h>
91 #include <linux/init.h>
92 #include <linux/io.h>
93 #include <linux/kernel.h>
94 #include <linux/sched/signal.h>
95 #include <linux/kmod.h>
96 #include <linux/list.h>
97 #include <linux/miscdevice.h>
98 #include <linux/module.h>
99 #include <linux/mutex.h>
100 #include <linux/net.h>
101 #include <linux/poll.h>
102 #include <linux/random.h>
103 #include <linux/skbuff.h>
104 #include <linux/smp.h>
105 #include <linux/socket.h>
106 #include <linux/stddef.h>
107 #include <linux/unistd.h>
108 #include <linux/wait.h>
109 #include <linux/workqueue.h>
110 #include <net/sock.h>
111 #include <net/af_vsock.h>
112 
113 static int __vsock_bind(struct sock *sk, struct sockaddr_vm *addr);
114 static void vsock_sk_destruct(struct sock *sk);
115 static int vsock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
116 
117 /* Protocol family. */
118 static struct proto vsock_proto = {
119 	.name = "AF_VSOCK",
120 	.owner = THIS_MODULE,
121 	.obj_size = sizeof(struct vsock_sock),
122 };
123 
124 /* The default peer timeout indicates how long we will wait for a peer response
125  * to a control message.
126  */
127 #define VSOCK_DEFAULT_CONNECT_TIMEOUT (2 * HZ)
128 
129 static const struct vsock_transport *transport;
130 static DEFINE_MUTEX(vsock_register_mutex);
131 
132 /**** EXPORTS ****/
133 
134 /* Get the ID of the local context.  This is transport dependent. */
135 
136 int vm_sockets_get_local_cid(void)
137 {
138 	return transport->get_local_cid();
139 }
140 EXPORT_SYMBOL_GPL(vm_sockets_get_local_cid);
141 
142 /**** UTILS ****/
143 
144 /* Each bound VSocket is stored in the bind hash table and each connected
145  * VSocket is stored in the connected hash table.
146  *
147  * Unbound sockets are all put on the same list attached to the end of the hash
148  * table (vsock_unbound_sockets).  Bound sockets are added to the hash table in
149  * the bucket that their local address hashes to (vsock_bound_sockets(addr)
150  * represents the list that addr hashes to).
151  *
152  * Specifically, we initialize the vsock_bind_table array to a size of
153  * VSOCK_HASH_SIZE + 1 so that vsock_bind_table[0] through
154  * vsock_bind_table[VSOCK_HASH_SIZE - 1] are for bound sockets and
155  * vsock_bind_table[VSOCK_HASH_SIZE] is for unbound sockets.  The hash function
156  * mods with VSOCK_HASH_SIZE to ensure this.
157  */
158 #define MAX_PORT_RETRIES        24
159 
160 #define VSOCK_HASH(addr)        ((addr)->svm_port % VSOCK_HASH_SIZE)
161 #define vsock_bound_sockets(addr) (&vsock_bind_table[VSOCK_HASH(addr)])
162 #define vsock_unbound_sockets     (&vsock_bind_table[VSOCK_HASH_SIZE])
163 
164 /* XXX This can probably be implemented in a better way. */
165 #define VSOCK_CONN_HASH(src, dst)				\
166 	(((src)->svm_cid ^ (dst)->svm_port) % VSOCK_HASH_SIZE)
167 #define vsock_connected_sockets(src, dst)		\
168 	(&vsock_connected_table[VSOCK_CONN_HASH(src, dst)])
169 #define vsock_connected_sockets_vsk(vsk)				\
170 	vsock_connected_sockets(&(vsk)->remote_addr, &(vsk)->local_addr)
171 
172 struct list_head vsock_bind_table[VSOCK_HASH_SIZE + 1];
173 EXPORT_SYMBOL_GPL(vsock_bind_table);
174 struct list_head vsock_connected_table[VSOCK_HASH_SIZE];
175 EXPORT_SYMBOL_GPL(vsock_connected_table);
176 DEFINE_SPINLOCK(vsock_table_lock);
177 EXPORT_SYMBOL_GPL(vsock_table_lock);
178 
179 /* Autobind this socket to the local address if necessary. */
180 static int vsock_auto_bind(struct vsock_sock *vsk)
181 {
182 	struct sock *sk = sk_vsock(vsk);
183 	struct sockaddr_vm local_addr;
184 
185 	if (vsock_addr_bound(&vsk->local_addr))
186 		return 0;
187 	vsock_addr_init(&local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
188 	return __vsock_bind(sk, &local_addr);
189 }
190 
191 static int __init vsock_init_tables(void)
192 {
193 	int i;
194 
195 	for (i = 0; i < ARRAY_SIZE(vsock_bind_table); i++)
196 		INIT_LIST_HEAD(&vsock_bind_table[i]);
197 
198 	for (i = 0; i < ARRAY_SIZE(vsock_connected_table); i++)
199 		INIT_LIST_HEAD(&vsock_connected_table[i]);
200 	return 0;
201 }
202 
203 static void __vsock_insert_bound(struct list_head *list,
204 				 struct vsock_sock *vsk)
205 {
206 	sock_hold(&vsk->sk);
207 	list_add(&vsk->bound_table, list);
208 }
209 
210 static void __vsock_insert_connected(struct list_head *list,
211 				     struct vsock_sock *vsk)
212 {
213 	sock_hold(&vsk->sk);
214 	list_add(&vsk->connected_table, list);
215 }
216 
217 static void __vsock_remove_bound(struct vsock_sock *vsk)
218 {
219 	list_del_init(&vsk->bound_table);
220 	sock_put(&vsk->sk);
221 }
222 
223 static void __vsock_remove_connected(struct vsock_sock *vsk)
224 {
225 	list_del_init(&vsk->connected_table);
226 	sock_put(&vsk->sk);
227 }
228 
229 static struct sock *__vsock_find_bound_socket(struct sockaddr_vm *addr)
230 {
231 	struct vsock_sock *vsk;
232 
233 	list_for_each_entry(vsk, vsock_bound_sockets(addr), bound_table)
234 		if (addr->svm_port == vsk->local_addr.svm_port)
235 			return sk_vsock(vsk);
236 
237 	return NULL;
238 }
239 
240 static struct sock *__vsock_find_connected_socket(struct sockaddr_vm *src,
241 						  struct sockaddr_vm *dst)
242 {
243 	struct vsock_sock *vsk;
244 
245 	list_for_each_entry(vsk, vsock_connected_sockets(src, dst),
246 			    connected_table) {
247 		if (vsock_addr_equals_addr(src, &vsk->remote_addr) &&
248 		    dst->svm_port == vsk->local_addr.svm_port) {
249 			return sk_vsock(vsk);
250 		}
251 	}
252 
253 	return NULL;
254 }
255 
256 static void vsock_insert_unbound(struct vsock_sock *vsk)
257 {
258 	spin_lock_bh(&vsock_table_lock);
259 	__vsock_insert_bound(vsock_unbound_sockets, vsk);
260 	spin_unlock_bh(&vsock_table_lock);
261 }
262 
263 void vsock_insert_connected(struct vsock_sock *vsk)
264 {
265 	struct list_head *list = vsock_connected_sockets(
266 		&vsk->remote_addr, &vsk->local_addr);
267 
268 	spin_lock_bh(&vsock_table_lock);
269 	__vsock_insert_connected(list, vsk);
270 	spin_unlock_bh(&vsock_table_lock);
271 }
272 EXPORT_SYMBOL_GPL(vsock_insert_connected);
273 
274 void vsock_remove_bound(struct vsock_sock *vsk)
275 {
276 	spin_lock_bh(&vsock_table_lock);
277 	if (__vsock_in_bound_table(vsk))
278 		__vsock_remove_bound(vsk);
279 	spin_unlock_bh(&vsock_table_lock);
280 }
281 EXPORT_SYMBOL_GPL(vsock_remove_bound);
282 
283 void vsock_remove_connected(struct vsock_sock *vsk)
284 {
285 	spin_lock_bh(&vsock_table_lock);
286 	if (__vsock_in_connected_table(vsk))
287 		__vsock_remove_connected(vsk);
288 	spin_unlock_bh(&vsock_table_lock);
289 }
290 EXPORT_SYMBOL_GPL(vsock_remove_connected);
291 
292 struct sock *vsock_find_bound_socket(struct sockaddr_vm *addr)
293 {
294 	struct sock *sk;
295 
296 	spin_lock_bh(&vsock_table_lock);
297 	sk = __vsock_find_bound_socket(addr);
298 	if (sk)
299 		sock_hold(sk);
300 
301 	spin_unlock_bh(&vsock_table_lock);
302 
303 	return sk;
304 }
305 EXPORT_SYMBOL_GPL(vsock_find_bound_socket);
306 
307 struct sock *vsock_find_connected_socket(struct sockaddr_vm *src,
308 					 struct sockaddr_vm *dst)
309 {
310 	struct sock *sk;
311 
312 	spin_lock_bh(&vsock_table_lock);
313 	sk = __vsock_find_connected_socket(src, dst);
314 	if (sk)
315 		sock_hold(sk);
316 
317 	spin_unlock_bh(&vsock_table_lock);
318 
319 	return sk;
320 }
321 EXPORT_SYMBOL_GPL(vsock_find_connected_socket);
322 
323 void vsock_remove_sock(struct vsock_sock *vsk)
324 {
325 	vsock_remove_bound(vsk);
326 	vsock_remove_connected(vsk);
327 }
328 EXPORT_SYMBOL_GPL(vsock_remove_sock);
329 
330 void vsock_for_each_connected_socket(void (*fn)(struct sock *sk))
331 {
332 	int i;
333 
334 	spin_lock_bh(&vsock_table_lock);
335 
336 	for (i = 0; i < ARRAY_SIZE(vsock_connected_table); i++) {
337 		struct vsock_sock *vsk;
338 		list_for_each_entry(vsk, &vsock_connected_table[i],
339 				    connected_table)
340 			fn(sk_vsock(vsk));
341 	}
342 
343 	spin_unlock_bh(&vsock_table_lock);
344 }
345 EXPORT_SYMBOL_GPL(vsock_for_each_connected_socket);
346 
347 void vsock_add_pending(struct sock *listener, struct sock *pending)
348 {
349 	struct vsock_sock *vlistener;
350 	struct vsock_sock *vpending;
351 
352 	vlistener = vsock_sk(listener);
353 	vpending = vsock_sk(pending);
354 
355 	sock_hold(pending);
356 	sock_hold(listener);
357 	list_add_tail(&vpending->pending_links, &vlistener->pending_links);
358 }
359 EXPORT_SYMBOL_GPL(vsock_add_pending);
360 
361 void vsock_remove_pending(struct sock *listener, struct sock *pending)
362 {
363 	struct vsock_sock *vpending = vsock_sk(pending);
364 
365 	list_del_init(&vpending->pending_links);
366 	sock_put(listener);
367 	sock_put(pending);
368 }
369 EXPORT_SYMBOL_GPL(vsock_remove_pending);
370 
371 void vsock_enqueue_accept(struct sock *listener, struct sock *connected)
372 {
373 	struct vsock_sock *vlistener;
374 	struct vsock_sock *vconnected;
375 
376 	vlistener = vsock_sk(listener);
377 	vconnected = vsock_sk(connected);
378 
379 	sock_hold(connected);
380 	sock_hold(listener);
381 	list_add_tail(&vconnected->accept_queue, &vlistener->accept_queue);
382 }
383 EXPORT_SYMBOL_GPL(vsock_enqueue_accept);
384 
385 static struct sock *vsock_dequeue_accept(struct sock *listener)
386 {
387 	struct vsock_sock *vlistener;
388 	struct vsock_sock *vconnected;
389 
390 	vlistener = vsock_sk(listener);
391 
392 	if (list_empty(&vlistener->accept_queue))
393 		return NULL;
394 
395 	vconnected = list_entry(vlistener->accept_queue.next,
396 				struct vsock_sock, accept_queue);
397 
398 	list_del_init(&vconnected->accept_queue);
399 	sock_put(listener);
400 	/* The caller will need a reference on the connected socket so we let
401 	 * it call sock_put().
402 	 */
403 
404 	return sk_vsock(vconnected);
405 }
406 
407 static bool vsock_is_accept_queue_empty(struct sock *sk)
408 {
409 	struct vsock_sock *vsk = vsock_sk(sk);
410 	return list_empty(&vsk->accept_queue);
411 }
412 
413 static bool vsock_is_pending(struct sock *sk)
414 {
415 	struct vsock_sock *vsk = vsock_sk(sk);
416 	return !list_empty(&vsk->pending_links);
417 }
418 
419 static int vsock_send_shutdown(struct sock *sk, int mode)
420 {
421 	return transport->shutdown(vsock_sk(sk), mode);
422 }
423 
424 static void vsock_pending_work(struct work_struct *work)
425 {
426 	struct sock *sk;
427 	struct sock *listener;
428 	struct vsock_sock *vsk;
429 	bool cleanup;
430 
431 	vsk = container_of(work, struct vsock_sock, pending_work.work);
432 	sk = sk_vsock(vsk);
433 	listener = vsk->listener;
434 	cleanup = true;
435 
436 	lock_sock(listener);
437 	lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
438 
439 	if (vsock_is_pending(sk)) {
440 		vsock_remove_pending(listener, sk);
441 
442 		listener->sk_ack_backlog--;
443 	} else if (!vsk->rejected) {
444 		/* We are not on the pending list and accept() did not reject
445 		 * us, so we must have been accepted by our user process.  We
446 		 * just need to drop our references to the sockets and be on
447 		 * our way.
448 		 */
449 		cleanup = false;
450 		goto out;
451 	}
452 
453 	/* We need to remove ourself from the global connected sockets list so
454 	 * incoming packets can't find this socket, and to reduce the reference
455 	 * count.
456 	 */
457 	vsock_remove_connected(vsk);
458 
459 	sk->sk_state = TCP_CLOSE;
460 
461 out:
462 	release_sock(sk);
463 	release_sock(listener);
464 	if (cleanup)
465 		sock_put(sk);
466 
467 	sock_put(sk);
468 	sock_put(listener);
469 }
470 
471 /**** SOCKET OPERATIONS ****/
472 
473 static int __vsock_bind_stream(struct vsock_sock *vsk,
474 			       struct sockaddr_vm *addr)
475 {
476 	static u32 port;
477 	struct sockaddr_vm new_addr;
478 
479 	if (!port)
480 		port = LAST_RESERVED_PORT + 1 +
481 			prandom_u32_max(U32_MAX - LAST_RESERVED_PORT);
482 
483 	vsock_addr_init(&new_addr, addr->svm_cid, addr->svm_port);
484 
485 	if (addr->svm_port == VMADDR_PORT_ANY) {
486 		bool found = false;
487 		unsigned int i;
488 
489 		for (i = 0; i < MAX_PORT_RETRIES; i++) {
490 			if (port <= LAST_RESERVED_PORT)
491 				port = LAST_RESERVED_PORT + 1;
492 
493 			new_addr.svm_port = port++;
494 
495 			if (!__vsock_find_bound_socket(&new_addr)) {
496 				found = true;
497 				break;
498 			}
499 		}
500 
501 		if (!found)
502 			return -EADDRNOTAVAIL;
503 	} else {
504 		/* If port is in reserved range, ensure caller
505 		 * has necessary privileges.
506 		 */
507 		if (addr->svm_port <= LAST_RESERVED_PORT &&
508 		    !capable(CAP_NET_BIND_SERVICE)) {
509 			return -EACCES;
510 		}
511 
512 		if (__vsock_find_bound_socket(&new_addr))
513 			return -EADDRINUSE;
514 	}
515 
516 	vsock_addr_init(&vsk->local_addr, new_addr.svm_cid, new_addr.svm_port);
517 
518 	/* Remove stream sockets from the unbound list and add them to the hash
519 	 * table for easy lookup by its address.  The unbound list is simply an
520 	 * extra entry at the end of the hash table, a trick used by AF_UNIX.
521 	 */
522 	__vsock_remove_bound(vsk);
523 	__vsock_insert_bound(vsock_bound_sockets(&vsk->local_addr), vsk);
524 
525 	return 0;
526 }
527 
528 static int __vsock_bind_dgram(struct vsock_sock *vsk,
529 			      struct sockaddr_vm *addr)
530 {
531 	return transport->dgram_bind(vsk, addr);
532 }
533 
534 static int __vsock_bind(struct sock *sk, struct sockaddr_vm *addr)
535 {
536 	struct vsock_sock *vsk = vsock_sk(sk);
537 	u32 cid;
538 	int retval;
539 
540 	/* First ensure this socket isn't already bound. */
541 	if (vsock_addr_bound(&vsk->local_addr))
542 		return -EINVAL;
543 
544 	/* Now bind to the provided address or select appropriate values if
545 	 * none are provided (VMADDR_CID_ANY and VMADDR_PORT_ANY).  Note that
546 	 * like AF_INET prevents binding to a non-local IP address (in most
547 	 * cases), we only allow binding to the local CID.
548 	 */
549 	cid = transport->get_local_cid();
550 	if (addr->svm_cid != cid && addr->svm_cid != VMADDR_CID_ANY)
551 		return -EADDRNOTAVAIL;
552 
553 	switch (sk->sk_socket->type) {
554 	case SOCK_STREAM:
555 		spin_lock_bh(&vsock_table_lock);
556 		retval = __vsock_bind_stream(vsk, addr);
557 		spin_unlock_bh(&vsock_table_lock);
558 		break;
559 
560 	case SOCK_DGRAM:
561 		retval = __vsock_bind_dgram(vsk, addr);
562 		break;
563 
564 	default:
565 		retval = -EINVAL;
566 		break;
567 	}
568 
569 	return retval;
570 }
571 
572 static void vsock_connect_timeout(struct work_struct *work);
573 
574 struct sock *__vsock_create(struct net *net,
575 			    struct socket *sock,
576 			    struct sock *parent,
577 			    gfp_t priority,
578 			    unsigned short type,
579 			    int kern)
580 {
581 	struct sock *sk;
582 	struct vsock_sock *psk;
583 	struct vsock_sock *vsk;
584 
585 	sk = sk_alloc(net, AF_VSOCK, priority, &vsock_proto, kern);
586 	if (!sk)
587 		return NULL;
588 
589 	sock_init_data(sock, sk);
590 
591 	/* sk->sk_type is normally set in sock_init_data, but only if sock is
592 	 * non-NULL. We make sure that our sockets always have a type by
593 	 * setting it here if needed.
594 	 */
595 	if (!sock)
596 		sk->sk_type = type;
597 
598 	vsk = vsock_sk(sk);
599 	vsock_addr_init(&vsk->local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
600 	vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
601 
602 	sk->sk_destruct = vsock_sk_destruct;
603 	sk->sk_backlog_rcv = vsock_queue_rcv_skb;
604 	sock_reset_flag(sk, SOCK_DONE);
605 
606 	INIT_LIST_HEAD(&vsk->bound_table);
607 	INIT_LIST_HEAD(&vsk->connected_table);
608 	vsk->listener = NULL;
609 	INIT_LIST_HEAD(&vsk->pending_links);
610 	INIT_LIST_HEAD(&vsk->accept_queue);
611 	vsk->rejected = false;
612 	vsk->sent_request = false;
613 	vsk->ignore_connecting_rst = false;
614 	vsk->peer_shutdown = 0;
615 	INIT_DELAYED_WORK(&vsk->connect_work, vsock_connect_timeout);
616 	INIT_DELAYED_WORK(&vsk->pending_work, vsock_pending_work);
617 
618 	psk = parent ? vsock_sk(parent) : NULL;
619 	if (parent) {
620 		vsk->trusted = psk->trusted;
621 		vsk->owner = get_cred(psk->owner);
622 		vsk->connect_timeout = psk->connect_timeout;
623 	} else {
624 		vsk->trusted = capable(CAP_NET_ADMIN);
625 		vsk->owner = get_current_cred();
626 		vsk->connect_timeout = VSOCK_DEFAULT_CONNECT_TIMEOUT;
627 	}
628 
629 	if (transport->init(vsk, psk) < 0) {
630 		sk_free(sk);
631 		return NULL;
632 	}
633 
634 	if (sock)
635 		vsock_insert_unbound(vsk);
636 
637 	return sk;
638 }
639 EXPORT_SYMBOL_GPL(__vsock_create);
640 
641 static void __vsock_release(struct sock *sk)
642 {
643 	if (sk) {
644 		struct sk_buff *skb;
645 		struct sock *pending;
646 		struct vsock_sock *vsk;
647 
648 		vsk = vsock_sk(sk);
649 		pending = NULL;	/* Compiler warning. */
650 
651 		transport->release(vsk);
652 
653 		lock_sock(sk);
654 		sock_orphan(sk);
655 		sk->sk_shutdown = SHUTDOWN_MASK;
656 
657 		while ((skb = skb_dequeue(&sk->sk_receive_queue)))
658 			kfree_skb(skb);
659 
660 		/* Clean up any sockets that never were accepted. */
661 		while ((pending = vsock_dequeue_accept(sk)) != NULL) {
662 			__vsock_release(pending);
663 			sock_put(pending);
664 		}
665 
666 		release_sock(sk);
667 		sock_put(sk);
668 	}
669 }
670 
671 static void vsock_sk_destruct(struct sock *sk)
672 {
673 	struct vsock_sock *vsk = vsock_sk(sk);
674 
675 	transport->destruct(vsk);
676 
677 	/* When clearing these addresses, there's no need to set the family and
678 	 * possibly register the address family with the kernel.
679 	 */
680 	vsock_addr_init(&vsk->local_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
681 	vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY, VMADDR_PORT_ANY);
682 
683 	put_cred(vsk->owner);
684 }
685 
686 static int vsock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
687 {
688 	int err;
689 
690 	err = sock_queue_rcv_skb(sk, skb);
691 	if (err)
692 		kfree_skb(skb);
693 
694 	return err;
695 }
696 
697 s64 vsock_stream_has_data(struct vsock_sock *vsk)
698 {
699 	return transport->stream_has_data(vsk);
700 }
701 EXPORT_SYMBOL_GPL(vsock_stream_has_data);
702 
703 s64 vsock_stream_has_space(struct vsock_sock *vsk)
704 {
705 	return transport->stream_has_space(vsk);
706 }
707 EXPORT_SYMBOL_GPL(vsock_stream_has_space);
708 
709 static int vsock_release(struct socket *sock)
710 {
711 	__vsock_release(sock->sk);
712 	sock->sk = NULL;
713 	sock->state = SS_FREE;
714 
715 	return 0;
716 }
717 
718 static int
719 vsock_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
720 {
721 	int err;
722 	struct sock *sk;
723 	struct sockaddr_vm *vm_addr;
724 
725 	sk = sock->sk;
726 
727 	if (vsock_addr_cast(addr, addr_len, &vm_addr) != 0)
728 		return -EINVAL;
729 
730 	lock_sock(sk);
731 	err = __vsock_bind(sk, vm_addr);
732 	release_sock(sk);
733 
734 	return err;
735 }
736 
737 static int vsock_getname(struct socket *sock,
738 			 struct sockaddr *addr, int peer)
739 {
740 	int err;
741 	struct sock *sk;
742 	struct vsock_sock *vsk;
743 	struct sockaddr_vm *vm_addr;
744 
745 	sk = sock->sk;
746 	vsk = vsock_sk(sk);
747 	err = 0;
748 
749 	lock_sock(sk);
750 
751 	if (peer) {
752 		if (sock->state != SS_CONNECTED) {
753 			err = -ENOTCONN;
754 			goto out;
755 		}
756 		vm_addr = &vsk->remote_addr;
757 	} else {
758 		vm_addr = &vsk->local_addr;
759 	}
760 
761 	if (!vm_addr) {
762 		err = -EINVAL;
763 		goto out;
764 	}
765 
766 	/* sys_getsockname() and sys_getpeername() pass us a
767 	 * MAX_SOCK_ADDR-sized buffer and don't set addr_len.  Unfortunately
768 	 * that macro is defined in socket.c instead of .h, so we hardcode its
769 	 * value here.
770 	 */
771 	BUILD_BUG_ON(sizeof(*vm_addr) > 128);
772 	memcpy(addr, vm_addr, sizeof(*vm_addr));
773 	err = sizeof(*vm_addr);
774 
775 out:
776 	release_sock(sk);
777 	return err;
778 }
779 
780 static int vsock_shutdown(struct socket *sock, int mode)
781 {
782 	int err;
783 	struct sock *sk;
784 
785 	/* User level uses SHUT_RD (0) and SHUT_WR (1), but the kernel uses
786 	 * RCV_SHUTDOWN (1) and SEND_SHUTDOWN (2), so we must increment mode
787 	 * here like the other address families do.  Note also that the
788 	 * increment makes SHUT_RDWR (2) into RCV_SHUTDOWN | SEND_SHUTDOWN (3),
789 	 * which is what we want.
790 	 */
791 	mode++;
792 
793 	if ((mode & ~SHUTDOWN_MASK) || !mode)
794 		return -EINVAL;
795 
796 	/* If this is a STREAM socket and it is not connected then bail out
797 	 * immediately.  If it is a DGRAM socket then we must first kick the
798 	 * socket so that it wakes up from any sleeping calls, for example
799 	 * recv(), and then afterwards return the error.
800 	 */
801 
802 	sk = sock->sk;
803 	if (sock->state == SS_UNCONNECTED) {
804 		err = -ENOTCONN;
805 		if (sk->sk_type == SOCK_STREAM)
806 			return err;
807 	} else {
808 		sock->state = SS_DISCONNECTING;
809 		err = 0;
810 	}
811 
812 	/* Receive and send shutdowns are treated alike. */
813 	mode = mode & (RCV_SHUTDOWN | SEND_SHUTDOWN);
814 	if (mode) {
815 		lock_sock(sk);
816 		sk->sk_shutdown |= mode;
817 		sk->sk_state_change(sk);
818 		release_sock(sk);
819 
820 		if (sk->sk_type == SOCK_STREAM) {
821 			sock_reset_flag(sk, SOCK_DONE);
822 			vsock_send_shutdown(sk, mode);
823 		}
824 	}
825 
826 	return err;
827 }
828 
829 static __poll_t vsock_poll(struct file *file, struct socket *sock,
830 			       poll_table *wait)
831 {
832 	struct sock *sk;
833 	__poll_t mask;
834 	struct vsock_sock *vsk;
835 
836 	sk = sock->sk;
837 	vsk = vsock_sk(sk);
838 
839 	poll_wait(file, sk_sleep(sk), wait);
840 	mask = 0;
841 
842 	if (sk->sk_err)
843 		/* Signify that there has been an error on this socket. */
844 		mask |= EPOLLERR;
845 
846 	/* INET sockets treat local write shutdown and peer write shutdown as a
847 	 * case of EPOLLHUP set.
848 	 */
849 	if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
850 	    ((sk->sk_shutdown & SEND_SHUTDOWN) &&
851 	     (vsk->peer_shutdown & SEND_SHUTDOWN))) {
852 		mask |= EPOLLHUP;
853 	}
854 
855 	if (sk->sk_shutdown & RCV_SHUTDOWN ||
856 	    vsk->peer_shutdown & SEND_SHUTDOWN) {
857 		mask |= EPOLLRDHUP;
858 	}
859 
860 	if (sock->type == SOCK_DGRAM) {
861 		/* For datagram sockets we can read if there is something in
862 		 * the queue and write as long as the socket isn't shutdown for
863 		 * sending.
864 		 */
865 		if (!skb_queue_empty(&sk->sk_receive_queue) ||
866 		    (sk->sk_shutdown & RCV_SHUTDOWN)) {
867 			mask |= EPOLLIN | EPOLLRDNORM;
868 		}
869 
870 		if (!(sk->sk_shutdown & SEND_SHUTDOWN))
871 			mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND;
872 
873 	} else if (sock->type == SOCK_STREAM) {
874 		lock_sock(sk);
875 
876 		/* Listening sockets that have connections in their accept
877 		 * queue can be read.
878 		 */
879 		if (sk->sk_state == TCP_LISTEN
880 		    && !vsock_is_accept_queue_empty(sk))
881 			mask |= EPOLLIN | EPOLLRDNORM;
882 
883 		/* If there is something in the queue then we can read. */
884 		if (transport->stream_is_active(vsk) &&
885 		    !(sk->sk_shutdown & RCV_SHUTDOWN)) {
886 			bool data_ready_now = false;
887 			int ret = transport->notify_poll_in(
888 					vsk, 1, &data_ready_now);
889 			if (ret < 0) {
890 				mask |= EPOLLERR;
891 			} else {
892 				if (data_ready_now)
893 					mask |= EPOLLIN | EPOLLRDNORM;
894 
895 			}
896 		}
897 
898 		/* Sockets whose connections have been closed, reset, or
899 		 * terminated should also be considered read, and we check the
900 		 * shutdown flag for that.
901 		 */
902 		if (sk->sk_shutdown & RCV_SHUTDOWN ||
903 		    vsk->peer_shutdown & SEND_SHUTDOWN) {
904 			mask |= EPOLLIN | EPOLLRDNORM;
905 		}
906 
907 		/* Connected sockets that can produce data can be written. */
908 		if (sk->sk_state == TCP_ESTABLISHED) {
909 			if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
910 				bool space_avail_now = false;
911 				int ret = transport->notify_poll_out(
912 						vsk, 1, &space_avail_now);
913 				if (ret < 0) {
914 					mask |= EPOLLERR;
915 				} else {
916 					if (space_avail_now)
917 						/* Remove EPOLLWRBAND since INET
918 						 * sockets are not setting it.
919 						 */
920 						mask |= EPOLLOUT | EPOLLWRNORM;
921 
922 				}
923 			}
924 		}
925 
926 		/* Simulate INET socket poll behaviors, which sets
927 		 * EPOLLOUT|EPOLLWRNORM when peer is closed and nothing to read,
928 		 * but local send is not shutdown.
929 		 */
930 		if (sk->sk_state == TCP_CLOSE || sk->sk_state == TCP_CLOSING) {
931 			if (!(sk->sk_shutdown & SEND_SHUTDOWN))
932 				mask |= EPOLLOUT | EPOLLWRNORM;
933 
934 		}
935 
936 		release_sock(sk);
937 	}
938 
939 	return mask;
940 }
941 
942 static int vsock_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
943 			       size_t len)
944 {
945 	int err;
946 	struct sock *sk;
947 	struct vsock_sock *vsk;
948 	struct sockaddr_vm *remote_addr;
949 
950 	if (msg->msg_flags & MSG_OOB)
951 		return -EOPNOTSUPP;
952 
953 	/* For now, MSG_DONTWAIT is always assumed... */
954 	err = 0;
955 	sk = sock->sk;
956 	vsk = vsock_sk(sk);
957 
958 	lock_sock(sk);
959 
960 	err = vsock_auto_bind(vsk);
961 	if (err)
962 		goto out;
963 
964 
965 	/* If the provided message contains an address, use that.  Otherwise
966 	 * fall back on the socket's remote handle (if it has been connected).
967 	 */
968 	if (msg->msg_name &&
969 	    vsock_addr_cast(msg->msg_name, msg->msg_namelen,
970 			    &remote_addr) == 0) {
971 		/* Ensure this address is of the right type and is a valid
972 		 * destination.
973 		 */
974 
975 		if (remote_addr->svm_cid == VMADDR_CID_ANY)
976 			remote_addr->svm_cid = transport->get_local_cid();
977 
978 		if (!vsock_addr_bound(remote_addr)) {
979 			err = -EINVAL;
980 			goto out;
981 		}
982 	} else if (sock->state == SS_CONNECTED) {
983 		remote_addr = &vsk->remote_addr;
984 
985 		if (remote_addr->svm_cid == VMADDR_CID_ANY)
986 			remote_addr->svm_cid = transport->get_local_cid();
987 
988 		/* XXX Should connect() or this function ensure remote_addr is
989 		 * bound?
990 		 */
991 		if (!vsock_addr_bound(&vsk->remote_addr)) {
992 			err = -EINVAL;
993 			goto out;
994 		}
995 	} else {
996 		err = -EINVAL;
997 		goto out;
998 	}
999 
1000 	if (!transport->dgram_allow(remote_addr->svm_cid,
1001 				    remote_addr->svm_port)) {
1002 		err = -EINVAL;
1003 		goto out;
1004 	}
1005 
1006 	err = transport->dgram_enqueue(vsk, remote_addr, msg, len);
1007 
1008 out:
1009 	release_sock(sk);
1010 	return err;
1011 }
1012 
1013 static int vsock_dgram_connect(struct socket *sock,
1014 			       struct sockaddr *addr, int addr_len, int flags)
1015 {
1016 	int err;
1017 	struct sock *sk;
1018 	struct vsock_sock *vsk;
1019 	struct sockaddr_vm *remote_addr;
1020 
1021 	sk = sock->sk;
1022 	vsk = vsock_sk(sk);
1023 
1024 	err = vsock_addr_cast(addr, addr_len, &remote_addr);
1025 	if (err == -EAFNOSUPPORT && remote_addr->svm_family == AF_UNSPEC) {
1026 		lock_sock(sk);
1027 		vsock_addr_init(&vsk->remote_addr, VMADDR_CID_ANY,
1028 				VMADDR_PORT_ANY);
1029 		sock->state = SS_UNCONNECTED;
1030 		release_sock(sk);
1031 		return 0;
1032 	} else if (err != 0)
1033 		return -EINVAL;
1034 
1035 	lock_sock(sk);
1036 
1037 	err = vsock_auto_bind(vsk);
1038 	if (err)
1039 		goto out;
1040 
1041 	if (!transport->dgram_allow(remote_addr->svm_cid,
1042 				    remote_addr->svm_port)) {
1043 		err = -EINVAL;
1044 		goto out;
1045 	}
1046 
1047 	memcpy(&vsk->remote_addr, remote_addr, sizeof(vsk->remote_addr));
1048 	sock->state = SS_CONNECTED;
1049 
1050 out:
1051 	release_sock(sk);
1052 	return err;
1053 }
1054 
1055 static int vsock_dgram_recvmsg(struct socket *sock, struct msghdr *msg,
1056 			       size_t len, int flags)
1057 {
1058 	return transport->dgram_dequeue(vsock_sk(sock->sk), msg, len, flags);
1059 }
1060 
1061 static const struct proto_ops vsock_dgram_ops = {
1062 	.family = PF_VSOCK,
1063 	.owner = THIS_MODULE,
1064 	.release = vsock_release,
1065 	.bind = vsock_bind,
1066 	.connect = vsock_dgram_connect,
1067 	.socketpair = sock_no_socketpair,
1068 	.accept = sock_no_accept,
1069 	.getname = vsock_getname,
1070 	.poll = vsock_poll,
1071 	.ioctl = sock_no_ioctl,
1072 	.listen = sock_no_listen,
1073 	.shutdown = vsock_shutdown,
1074 	.setsockopt = sock_no_setsockopt,
1075 	.getsockopt = sock_no_getsockopt,
1076 	.sendmsg = vsock_dgram_sendmsg,
1077 	.recvmsg = vsock_dgram_recvmsg,
1078 	.mmap = sock_no_mmap,
1079 	.sendpage = sock_no_sendpage,
1080 };
1081 
1082 static int vsock_transport_cancel_pkt(struct vsock_sock *vsk)
1083 {
1084 	if (!transport->cancel_pkt)
1085 		return -EOPNOTSUPP;
1086 
1087 	return transport->cancel_pkt(vsk);
1088 }
1089 
1090 static void vsock_connect_timeout(struct work_struct *work)
1091 {
1092 	struct sock *sk;
1093 	struct vsock_sock *vsk;
1094 	int cancel = 0;
1095 
1096 	vsk = container_of(work, struct vsock_sock, connect_work.work);
1097 	sk = sk_vsock(vsk);
1098 
1099 	lock_sock(sk);
1100 	if (sk->sk_state == TCP_SYN_SENT &&
1101 	    (sk->sk_shutdown != SHUTDOWN_MASK)) {
1102 		sk->sk_state = TCP_CLOSE;
1103 		sk->sk_err = ETIMEDOUT;
1104 		sk->sk_error_report(sk);
1105 		cancel = 1;
1106 	}
1107 	release_sock(sk);
1108 	if (cancel)
1109 		vsock_transport_cancel_pkt(vsk);
1110 
1111 	sock_put(sk);
1112 }
1113 
1114 static int vsock_stream_connect(struct socket *sock, struct sockaddr *addr,
1115 				int addr_len, int flags)
1116 {
1117 	int err;
1118 	struct sock *sk;
1119 	struct vsock_sock *vsk;
1120 	struct sockaddr_vm *remote_addr;
1121 	long timeout;
1122 	DEFINE_WAIT(wait);
1123 
1124 	err = 0;
1125 	sk = sock->sk;
1126 	vsk = vsock_sk(sk);
1127 
1128 	lock_sock(sk);
1129 
1130 	/* XXX AF_UNSPEC should make us disconnect like AF_INET. */
1131 	switch (sock->state) {
1132 	case SS_CONNECTED:
1133 		err = -EISCONN;
1134 		goto out;
1135 	case SS_DISCONNECTING:
1136 		err = -EINVAL;
1137 		goto out;
1138 	case SS_CONNECTING:
1139 		/* This continues on so we can move sock into the SS_CONNECTED
1140 		 * state once the connection has completed (at which point err
1141 		 * will be set to zero also).  Otherwise, we will either wait
1142 		 * for the connection or return -EALREADY should this be a
1143 		 * non-blocking call.
1144 		 */
1145 		err = -EALREADY;
1146 		break;
1147 	default:
1148 		if ((sk->sk_state == TCP_LISTEN) ||
1149 		    vsock_addr_cast(addr, addr_len, &remote_addr) != 0) {
1150 			err = -EINVAL;
1151 			goto out;
1152 		}
1153 
1154 		/* The hypervisor and well-known contexts do not have socket
1155 		 * endpoints.
1156 		 */
1157 		if (!transport->stream_allow(remote_addr->svm_cid,
1158 					     remote_addr->svm_port)) {
1159 			err = -ENETUNREACH;
1160 			goto out;
1161 		}
1162 
1163 		/* Set the remote address that we are connecting to. */
1164 		memcpy(&vsk->remote_addr, remote_addr,
1165 		       sizeof(vsk->remote_addr));
1166 
1167 		err = vsock_auto_bind(vsk);
1168 		if (err)
1169 			goto out;
1170 
1171 		sk->sk_state = TCP_SYN_SENT;
1172 
1173 		err = transport->connect(vsk);
1174 		if (err < 0)
1175 			goto out;
1176 
1177 		/* Mark sock as connecting and set the error code to in
1178 		 * progress in case this is a non-blocking connect.
1179 		 */
1180 		sock->state = SS_CONNECTING;
1181 		err = -EINPROGRESS;
1182 	}
1183 
1184 	/* The receive path will handle all communication until we are able to
1185 	 * enter the connected state.  Here we wait for the connection to be
1186 	 * completed or a notification of an error.
1187 	 */
1188 	timeout = vsk->connect_timeout;
1189 	prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1190 
1191 	while (sk->sk_state != TCP_ESTABLISHED && sk->sk_err == 0) {
1192 		if (flags & O_NONBLOCK) {
1193 			/* If we're not going to block, we schedule a timeout
1194 			 * function to generate a timeout on the connection
1195 			 * attempt, in case the peer doesn't respond in a
1196 			 * timely manner. We hold on to the socket until the
1197 			 * timeout fires.
1198 			 */
1199 			sock_hold(sk);
1200 			schedule_delayed_work(&vsk->connect_work, timeout);
1201 
1202 			/* Skip ahead to preserve error code set above. */
1203 			goto out_wait;
1204 		}
1205 
1206 		release_sock(sk);
1207 		timeout = schedule_timeout(timeout);
1208 		lock_sock(sk);
1209 
1210 		if (signal_pending(current)) {
1211 			err = sock_intr_errno(timeout);
1212 			sk->sk_state = TCP_CLOSE;
1213 			sock->state = SS_UNCONNECTED;
1214 			vsock_transport_cancel_pkt(vsk);
1215 			goto out_wait;
1216 		} else if (timeout == 0) {
1217 			err = -ETIMEDOUT;
1218 			sk->sk_state = TCP_CLOSE;
1219 			sock->state = SS_UNCONNECTED;
1220 			vsock_transport_cancel_pkt(vsk);
1221 			goto out_wait;
1222 		}
1223 
1224 		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1225 	}
1226 
1227 	if (sk->sk_err) {
1228 		err = -sk->sk_err;
1229 		sk->sk_state = TCP_CLOSE;
1230 		sock->state = SS_UNCONNECTED;
1231 	} else {
1232 		err = 0;
1233 	}
1234 
1235 out_wait:
1236 	finish_wait(sk_sleep(sk), &wait);
1237 out:
1238 	release_sock(sk);
1239 	return err;
1240 }
1241 
1242 static int vsock_accept(struct socket *sock, struct socket *newsock, int flags,
1243 			bool kern)
1244 {
1245 	struct sock *listener;
1246 	int err;
1247 	struct sock *connected;
1248 	struct vsock_sock *vconnected;
1249 	long timeout;
1250 	DEFINE_WAIT(wait);
1251 
1252 	err = 0;
1253 	listener = sock->sk;
1254 
1255 	lock_sock(listener);
1256 
1257 	if (sock->type != SOCK_STREAM) {
1258 		err = -EOPNOTSUPP;
1259 		goto out;
1260 	}
1261 
1262 	if (listener->sk_state != TCP_LISTEN) {
1263 		err = -EINVAL;
1264 		goto out;
1265 	}
1266 
1267 	/* Wait for children sockets to appear; these are the new sockets
1268 	 * created upon connection establishment.
1269 	 */
1270 	timeout = sock_sndtimeo(listener, flags & O_NONBLOCK);
1271 	prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE);
1272 
1273 	while ((connected = vsock_dequeue_accept(listener)) == NULL &&
1274 	       listener->sk_err == 0) {
1275 		release_sock(listener);
1276 		timeout = schedule_timeout(timeout);
1277 		finish_wait(sk_sleep(listener), &wait);
1278 		lock_sock(listener);
1279 
1280 		if (signal_pending(current)) {
1281 			err = sock_intr_errno(timeout);
1282 			goto out;
1283 		} else if (timeout == 0) {
1284 			err = -EAGAIN;
1285 			goto out;
1286 		}
1287 
1288 		prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE);
1289 	}
1290 	finish_wait(sk_sleep(listener), &wait);
1291 
1292 	if (listener->sk_err)
1293 		err = -listener->sk_err;
1294 
1295 	if (connected) {
1296 		listener->sk_ack_backlog--;
1297 
1298 		lock_sock_nested(connected, SINGLE_DEPTH_NESTING);
1299 		vconnected = vsock_sk(connected);
1300 
1301 		/* If the listener socket has received an error, then we should
1302 		 * reject this socket and return.  Note that we simply mark the
1303 		 * socket rejected, drop our reference, and let the cleanup
1304 		 * function handle the cleanup; the fact that we found it in
1305 		 * the listener's accept queue guarantees that the cleanup
1306 		 * function hasn't run yet.
1307 		 */
1308 		if (err) {
1309 			vconnected->rejected = true;
1310 		} else {
1311 			newsock->state = SS_CONNECTED;
1312 			sock_graft(connected, newsock);
1313 		}
1314 
1315 		release_sock(connected);
1316 		sock_put(connected);
1317 	}
1318 
1319 out:
1320 	release_sock(listener);
1321 	return err;
1322 }
1323 
1324 static int vsock_listen(struct socket *sock, int backlog)
1325 {
1326 	int err;
1327 	struct sock *sk;
1328 	struct vsock_sock *vsk;
1329 
1330 	sk = sock->sk;
1331 
1332 	lock_sock(sk);
1333 
1334 	if (sock->type != SOCK_STREAM) {
1335 		err = -EOPNOTSUPP;
1336 		goto out;
1337 	}
1338 
1339 	if (sock->state != SS_UNCONNECTED) {
1340 		err = -EINVAL;
1341 		goto out;
1342 	}
1343 
1344 	vsk = vsock_sk(sk);
1345 
1346 	if (!vsock_addr_bound(&vsk->local_addr)) {
1347 		err = -EINVAL;
1348 		goto out;
1349 	}
1350 
1351 	sk->sk_max_ack_backlog = backlog;
1352 	sk->sk_state = TCP_LISTEN;
1353 
1354 	err = 0;
1355 
1356 out:
1357 	release_sock(sk);
1358 	return err;
1359 }
1360 
1361 static int vsock_stream_setsockopt(struct socket *sock,
1362 				   int level,
1363 				   int optname,
1364 				   char __user *optval,
1365 				   unsigned int optlen)
1366 {
1367 	int err;
1368 	struct sock *sk;
1369 	struct vsock_sock *vsk;
1370 	u64 val;
1371 
1372 	if (level != AF_VSOCK)
1373 		return -ENOPROTOOPT;
1374 
1375 #define COPY_IN(_v)                                       \
1376 	do {						  \
1377 		if (optlen < sizeof(_v)) {		  \
1378 			err = -EINVAL;			  \
1379 			goto exit;			  \
1380 		}					  \
1381 		if (copy_from_user(&_v, optval, sizeof(_v)) != 0) {	\
1382 			err = -EFAULT;					\
1383 			goto exit;					\
1384 		}							\
1385 	} while (0)
1386 
1387 	err = 0;
1388 	sk = sock->sk;
1389 	vsk = vsock_sk(sk);
1390 
1391 	lock_sock(sk);
1392 
1393 	switch (optname) {
1394 	case SO_VM_SOCKETS_BUFFER_SIZE:
1395 		COPY_IN(val);
1396 		transport->set_buffer_size(vsk, val);
1397 		break;
1398 
1399 	case SO_VM_SOCKETS_BUFFER_MAX_SIZE:
1400 		COPY_IN(val);
1401 		transport->set_max_buffer_size(vsk, val);
1402 		break;
1403 
1404 	case SO_VM_SOCKETS_BUFFER_MIN_SIZE:
1405 		COPY_IN(val);
1406 		transport->set_min_buffer_size(vsk, val);
1407 		break;
1408 
1409 	case SO_VM_SOCKETS_CONNECT_TIMEOUT: {
1410 		struct __kernel_old_timeval tv;
1411 		COPY_IN(tv);
1412 		if (tv.tv_sec >= 0 && tv.tv_usec < USEC_PER_SEC &&
1413 		    tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1)) {
1414 			vsk->connect_timeout = tv.tv_sec * HZ +
1415 			    DIV_ROUND_UP(tv.tv_usec, (1000000 / HZ));
1416 			if (vsk->connect_timeout == 0)
1417 				vsk->connect_timeout =
1418 				    VSOCK_DEFAULT_CONNECT_TIMEOUT;
1419 
1420 		} else {
1421 			err = -ERANGE;
1422 		}
1423 		break;
1424 	}
1425 
1426 	default:
1427 		err = -ENOPROTOOPT;
1428 		break;
1429 	}
1430 
1431 #undef COPY_IN
1432 
1433 exit:
1434 	release_sock(sk);
1435 	return err;
1436 }
1437 
1438 static int vsock_stream_getsockopt(struct socket *sock,
1439 				   int level, int optname,
1440 				   char __user *optval,
1441 				   int __user *optlen)
1442 {
1443 	int err;
1444 	int len;
1445 	struct sock *sk;
1446 	struct vsock_sock *vsk;
1447 	u64 val;
1448 
1449 	if (level != AF_VSOCK)
1450 		return -ENOPROTOOPT;
1451 
1452 	err = get_user(len, optlen);
1453 	if (err != 0)
1454 		return err;
1455 
1456 #define COPY_OUT(_v)                            \
1457 	do {					\
1458 		if (len < sizeof(_v))		\
1459 			return -EINVAL;		\
1460 						\
1461 		len = sizeof(_v);		\
1462 		if (copy_to_user(optval, &_v, len) != 0)	\
1463 			return -EFAULT;				\
1464 								\
1465 	} while (0)
1466 
1467 	err = 0;
1468 	sk = sock->sk;
1469 	vsk = vsock_sk(sk);
1470 
1471 	switch (optname) {
1472 	case SO_VM_SOCKETS_BUFFER_SIZE:
1473 		val = transport->get_buffer_size(vsk);
1474 		COPY_OUT(val);
1475 		break;
1476 
1477 	case SO_VM_SOCKETS_BUFFER_MAX_SIZE:
1478 		val = transport->get_max_buffer_size(vsk);
1479 		COPY_OUT(val);
1480 		break;
1481 
1482 	case SO_VM_SOCKETS_BUFFER_MIN_SIZE:
1483 		val = transport->get_min_buffer_size(vsk);
1484 		COPY_OUT(val);
1485 		break;
1486 
1487 	case SO_VM_SOCKETS_CONNECT_TIMEOUT: {
1488 		struct __kernel_old_timeval tv;
1489 		tv.tv_sec = vsk->connect_timeout / HZ;
1490 		tv.tv_usec =
1491 		    (vsk->connect_timeout -
1492 		     tv.tv_sec * HZ) * (1000000 / HZ);
1493 		COPY_OUT(tv);
1494 		break;
1495 	}
1496 	default:
1497 		return -ENOPROTOOPT;
1498 	}
1499 
1500 	err = put_user(len, optlen);
1501 	if (err != 0)
1502 		return -EFAULT;
1503 
1504 #undef COPY_OUT
1505 
1506 	return 0;
1507 }
1508 
1509 static int vsock_stream_sendmsg(struct socket *sock, struct msghdr *msg,
1510 				size_t len)
1511 {
1512 	struct sock *sk;
1513 	struct vsock_sock *vsk;
1514 	ssize_t total_written;
1515 	long timeout;
1516 	int err;
1517 	struct vsock_transport_send_notify_data send_data;
1518 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
1519 
1520 	sk = sock->sk;
1521 	vsk = vsock_sk(sk);
1522 	total_written = 0;
1523 	err = 0;
1524 
1525 	if (msg->msg_flags & MSG_OOB)
1526 		return -EOPNOTSUPP;
1527 
1528 	lock_sock(sk);
1529 
1530 	/* Callers should not provide a destination with stream sockets. */
1531 	if (msg->msg_namelen) {
1532 		err = sk->sk_state == TCP_ESTABLISHED ? -EISCONN : -EOPNOTSUPP;
1533 		goto out;
1534 	}
1535 
1536 	/* Send data only if both sides are not shutdown in the direction. */
1537 	if (sk->sk_shutdown & SEND_SHUTDOWN ||
1538 	    vsk->peer_shutdown & RCV_SHUTDOWN) {
1539 		err = -EPIPE;
1540 		goto out;
1541 	}
1542 
1543 	if (sk->sk_state != TCP_ESTABLISHED ||
1544 	    !vsock_addr_bound(&vsk->local_addr)) {
1545 		err = -ENOTCONN;
1546 		goto out;
1547 	}
1548 
1549 	if (!vsock_addr_bound(&vsk->remote_addr)) {
1550 		err = -EDESTADDRREQ;
1551 		goto out;
1552 	}
1553 
1554 	/* Wait for room in the produce queue to enqueue our user's data. */
1555 	timeout = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
1556 
1557 	err = transport->notify_send_init(vsk, &send_data);
1558 	if (err < 0)
1559 		goto out;
1560 
1561 	while (total_written < len) {
1562 		ssize_t written;
1563 
1564 		add_wait_queue(sk_sleep(sk), &wait);
1565 		while (vsock_stream_has_space(vsk) == 0 &&
1566 		       sk->sk_err == 0 &&
1567 		       !(sk->sk_shutdown & SEND_SHUTDOWN) &&
1568 		       !(vsk->peer_shutdown & RCV_SHUTDOWN)) {
1569 
1570 			/* Don't wait for non-blocking sockets. */
1571 			if (timeout == 0) {
1572 				err = -EAGAIN;
1573 				remove_wait_queue(sk_sleep(sk), &wait);
1574 				goto out_err;
1575 			}
1576 
1577 			err = transport->notify_send_pre_block(vsk, &send_data);
1578 			if (err < 0) {
1579 				remove_wait_queue(sk_sleep(sk), &wait);
1580 				goto out_err;
1581 			}
1582 
1583 			release_sock(sk);
1584 			timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, timeout);
1585 			lock_sock(sk);
1586 			if (signal_pending(current)) {
1587 				err = sock_intr_errno(timeout);
1588 				remove_wait_queue(sk_sleep(sk), &wait);
1589 				goto out_err;
1590 			} else if (timeout == 0) {
1591 				err = -EAGAIN;
1592 				remove_wait_queue(sk_sleep(sk), &wait);
1593 				goto out_err;
1594 			}
1595 		}
1596 		remove_wait_queue(sk_sleep(sk), &wait);
1597 
1598 		/* These checks occur both as part of and after the loop
1599 		 * conditional since we need to check before and after
1600 		 * sleeping.
1601 		 */
1602 		if (sk->sk_err) {
1603 			err = -sk->sk_err;
1604 			goto out_err;
1605 		} else if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1606 			   (vsk->peer_shutdown & RCV_SHUTDOWN)) {
1607 			err = -EPIPE;
1608 			goto out_err;
1609 		}
1610 
1611 		err = transport->notify_send_pre_enqueue(vsk, &send_data);
1612 		if (err < 0)
1613 			goto out_err;
1614 
1615 		/* Note that enqueue will only write as many bytes as are free
1616 		 * in the produce queue, so we don't need to ensure len is
1617 		 * smaller than the queue size.  It is the caller's
1618 		 * responsibility to check how many bytes we were able to send.
1619 		 */
1620 
1621 		written = transport->stream_enqueue(
1622 				vsk, msg,
1623 				len - total_written);
1624 		if (written < 0) {
1625 			err = -ENOMEM;
1626 			goto out_err;
1627 		}
1628 
1629 		total_written += written;
1630 
1631 		err = transport->notify_send_post_enqueue(
1632 				vsk, written, &send_data);
1633 		if (err < 0)
1634 			goto out_err;
1635 
1636 	}
1637 
1638 out_err:
1639 	if (total_written > 0)
1640 		err = total_written;
1641 out:
1642 	release_sock(sk);
1643 	return err;
1644 }
1645 
1646 
1647 static int
1648 vsock_stream_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1649 		     int flags)
1650 {
1651 	struct sock *sk;
1652 	struct vsock_sock *vsk;
1653 	int err;
1654 	size_t target;
1655 	ssize_t copied;
1656 	long timeout;
1657 	struct vsock_transport_recv_notify_data recv_data;
1658 
1659 	DEFINE_WAIT(wait);
1660 
1661 	sk = sock->sk;
1662 	vsk = vsock_sk(sk);
1663 	err = 0;
1664 
1665 	lock_sock(sk);
1666 
1667 	if (sk->sk_state != TCP_ESTABLISHED) {
1668 		/* Recvmsg is supposed to return 0 if a peer performs an
1669 		 * orderly shutdown. Differentiate between that case and when a
1670 		 * peer has not connected or a local shutdown occured with the
1671 		 * SOCK_DONE flag.
1672 		 */
1673 		if (sock_flag(sk, SOCK_DONE))
1674 			err = 0;
1675 		else
1676 			err = -ENOTCONN;
1677 
1678 		goto out;
1679 	}
1680 
1681 	if (flags & MSG_OOB) {
1682 		err = -EOPNOTSUPP;
1683 		goto out;
1684 	}
1685 
1686 	/* We don't check peer_shutdown flag here since peer may actually shut
1687 	 * down, but there can be data in the queue that a local socket can
1688 	 * receive.
1689 	 */
1690 	if (sk->sk_shutdown & RCV_SHUTDOWN) {
1691 		err = 0;
1692 		goto out;
1693 	}
1694 
1695 	/* It is valid on Linux to pass in a zero-length receive buffer.  This
1696 	 * is not an error.  We may as well bail out now.
1697 	 */
1698 	if (!len) {
1699 		err = 0;
1700 		goto out;
1701 	}
1702 
1703 	/* We must not copy less than target bytes into the user's buffer
1704 	 * before returning successfully, so we wait for the consume queue to
1705 	 * have that much data to consume before dequeueing.  Note that this
1706 	 * makes it impossible to handle cases where target is greater than the
1707 	 * queue size.
1708 	 */
1709 	target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1710 	if (target >= transport->stream_rcvhiwat(vsk)) {
1711 		err = -ENOMEM;
1712 		goto out;
1713 	}
1714 	timeout = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1715 	copied = 0;
1716 
1717 	err = transport->notify_recv_init(vsk, target, &recv_data);
1718 	if (err < 0)
1719 		goto out;
1720 
1721 
1722 	while (1) {
1723 		s64 ready;
1724 
1725 		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
1726 		ready = vsock_stream_has_data(vsk);
1727 
1728 		if (ready == 0) {
1729 			if (sk->sk_err != 0 ||
1730 			    (sk->sk_shutdown & RCV_SHUTDOWN) ||
1731 			    (vsk->peer_shutdown & SEND_SHUTDOWN)) {
1732 				finish_wait(sk_sleep(sk), &wait);
1733 				break;
1734 			}
1735 			/* Don't wait for non-blocking sockets. */
1736 			if (timeout == 0) {
1737 				err = -EAGAIN;
1738 				finish_wait(sk_sleep(sk), &wait);
1739 				break;
1740 			}
1741 
1742 			err = transport->notify_recv_pre_block(
1743 					vsk, target, &recv_data);
1744 			if (err < 0) {
1745 				finish_wait(sk_sleep(sk), &wait);
1746 				break;
1747 			}
1748 			release_sock(sk);
1749 			timeout = schedule_timeout(timeout);
1750 			lock_sock(sk);
1751 
1752 			if (signal_pending(current)) {
1753 				err = sock_intr_errno(timeout);
1754 				finish_wait(sk_sleep(sk), &wait);
1755 				break;
1756 			} else if (timeout == 0) {
1757 				err = -EAGAIN;
1758 				finish_wait(sk_sleep(sk), &wait);
1759 				break;
1760 			}
1761 		} else {
1762 			ssize_t read;
1763 
1764 			finish_wait(sk_sleep(sk), &wait);
1765 
1766 			if (ready < 0) {
1767 				/* Invalid queue pair content. XXX This should
1768 				* be changed to a connection reset in a later
1769 				* change.
1770 				*/
1771 
1772 				err = -ENOMEM;
1773 				goto out;
1774 			}
1775 
1776 			err = transport->notify_recv_pre_dequeue(
1777 					vsk, target, &recv_data);
1778 			if (err < 0)
1779 				break;
1780 
1781 			read = transport->stream_dequeue(
1782 					vsk, msg,
1783 					len - copied, flags);
1784 			if (read < 0) {
1785 				err = -ENOMEM;
1786 				break;
1787 			}
1788 
1789 			copied += read;
1790 
1791 			err = transport->notify_recv_post_dequeue(
1792 					vsk, target, read,
1793 					!(flags & MSG_PEEK), &recv_data);
1794 			if (err < 0)
1795 				goto out;
1796 
1797 			if (read >= target || flags & MSG_PEEK)
1798 				break;
1799 
1800 			target -= read;
1801 		}
1802 	}
1803 
1804 	if (sk->sk_err)
1805 		err = -sk->sk_err;
1806 	else if (sk->sk_shutdown & RCV_SHUTDOWN)
1807 		err = 0;
1808 
1809 	if (copied > 0)
1810 		err = copied;
1811 
1812 out:
1813 	release_sock(sk);
1814 	return err;
1815 }
1816 
1817 static const struct proto_ops vsock_stream_ops = {
1818 	.family = PF_VSOCK,
1819 	.owner = THIS_MODULE,
1820 	.release = vsock_release,
1821 	.bind = vsock_bind,
1822 	.connect = vsock_stream_connect,
1823 	.socketpair = sock_no_socketpair,
1824 	.accept = vsock_accept,
1825 	.getname = vsock_getname,
1826 	.poll = vsock_poll,
1827 	.ioctl = sock_no_ioctl,
1828 	.listen = vsock_listen,
1829 	.shutdown = vsock_shutdown,
1830 	.setsockopt = vsock_stream_setsockopt,
1831 	.getsockopt = vsock_stream_getsockopt,
1832 	.sendmsg = vsock_stream_sendmsg,
1833 	.recvmsg = vsock_stream_recvmsg,
1834 	.mmap = sock_no_mmap,
1835 	.sendpage = sock_no_sendpage,
1836 };
1837 
1838 static int vsock_create(struct net *net, struct socket *sock,
1839 			int protocol, int kern)
1840 {
1841 	if (!sock)
1842 		return -EINVAL;
1843 
1844 	if (protocol && protocol != PF_VSOCK)
1845 		return -EPROTONOSUPPORT;
1846 
1847 	switch (sock->type) {
1848 	case SOCK_DGRAM:
1849 		sock->ops = &vsock_dgram_ops;
1850 		break;
1851 	case SOCK_STREAM:
1852 		sock->ops = &vsock_stream_ops;
1853 		break;
1854 	default:
1855 		return -ESOCKTNOSUPPORT;
1856 	}
1857 
1858 	sock->state = SS_UNCONNECTED;
1859 
1860 	return __vsock_create(net, sock, NULL, GFP_KERNEL, 0, kern) ? 0 : -ENOMEM;
1861 }
1862 
1863 static const struct net_proto_family vsock_family_ops = {
1864 	.family = AF_VSOCK,
1865 	.create = vsock_create,
1866 	.owner = THIS_MODULE,
1867 };
1868 
1869 static long vsock_dev_do_ioctl(struct file *filp,
1870 			       unsigned int cmd, void __user *ptr)
1871 {
1872 	u32 __user *p = ptr;
1873 	int retval = 0;
1874 
1875 	switch (cmd) {
1876 	case IOCTL_VM_SOCKETS_GET_LOCAL_CID:
1877 		if (put_user(transport->get_local_cid(), p) != 0)
1878 			retval = -EFAULT;
1879 		break;
1880 
1881 	default:
1882 		pr_err("Unknown ioctl %d\n", cmd);
1883 		retval = -EINVAL;
1884 	}
1885 
1886 	return retval;
1887 }
1888 
1889 static long vsock_dev_ioctl(struct file *filp,
1890 			    unsigned int cmd, unsigned long arg)
1891 {
1892 	return vsock_dev_do_ioctl(filp, cmd, (void __user *)arg);
1893 }
1894 
1895 #ifdef CONFIG_COMPAT
1896 static long vsock_dev_compat_ioctl(struct file *filp,
1897 				   unsigned int cmd, unsigned long arg)
1898 {
1899 	return vsock_dev_do_ioctl(filp, cmd, compat_ptr(arg));
1900 }
1901 #endif
1902 
1903 static const struct file_operations vsock_device_ops = {
1904 	.owner		= THIS_MODULE,
1905 	.unlocked_ioctl	= vsock_dev_ioctl,
1906 #ifdef CONFIG_COMPAT
1907 	.compat_ioctl	= vsock_dev_compat_ioctl,
1908 #endif
1909 	.open		= nonseekable_open,
1910 };
1911 
1912 static struct miscdevice vsock_device = {
1913 	.name		= "vsock",
1914 	.fops		= &vsock_device_ops,
1915 };
1916 
1917 int __vsock_core_init(const struct vsock_transport *t, struct module *owner)
1918 {
1919 	int err = mutex_lock_interruptible(&vsock_register_mutex);
1920 
1921 	if (err)
1922 		return err;
1923 
1924 	if (transport) {
1925 		err = -EBUSY;
1926 		goto err_busy;
1927 	}
1928 
1929 	/* Transport must be the owner of the protocol so that it can't
1930 	 * unload while there are open sockets.
1931 	 */
1932 	vsock_proto.owner = owner;
1933 	transport = t;
1934 
1935 	vsock_device.minor = MISC_DYNAMIC_MINOR;
1936 	err = misc_register(&vsock_device);
1937 	if (err) {
1938 		pr_err("Failed to register misc device\n");
1939 		goto err_reset_transport;
1940 	}
1941 
1942 	err = proto_register(&vsock_proto, 1);	/* we want our slab */
1943 	if (err) {
1944 		pr_err("Cannot register vsock protocol\n");
1945 		goto err_deregister_misc;
1946 	}
1947 
1948 	err = sock_register(&vsock_family_ops);
1949 	if (err) {
1950 		pr_err("could not register af_vsock (%d) address family: %d\n",
1951 		       AF_VSOCK, err);
1952 		goto err_unregister_proto;
1953 	}
1954 
1955 	mutex_unlock(&vsock_register_mutex);
1956 	return 0;
1957 
1958 err_unregister_proto:
1959 	proto_unregister(&vsock_proto);
1960 err_deregister_misc:
1961 	misc_deregister(&vsock_device);
1962 err_reset_transport:
1963 	transport = NULL;
1964 err_busy:
1965 	mutex_unlock(&vsock_register_mutex);
1966 	return err;
1967 }
1968 EXPORT_SYMBOL_GPL(__vsock_core_init);
1969 
1970 void vsock_core_exit(void)
1971 {
1972 	mutex_lock(&vsock_register_mutex);
1973 
1974 	misc_deregister(&vsock_device);
1975 	sock_unregister(AF_VSOCK);
1976 	proto_unregister(&vsock_proto);
1977 
1978 	/* We do not want the assignment below re-ordered. */
1979 	mb();
1980 	transport = NULL;
1981 
1982 	mutex_unlock(&vsock_register_mutex);
1983 }
1984 EXPORT_SYMBOL_GPL(vsock_core_exit);
1985 
1986 const struct vsock_transport *vsock_core_get_transport(void)
1987 {
1988 	/* vsock_register_mutex not taken since only the transport uses this
1989 	 * function and only while registered.
1990 	 */
1991 	return transport;
1992 }
1993 EXPORT_SYMBOL_GPL(vsock_core_get_transport);
1994 
1995 static void __exit vsock_exit(void)
1996 {
1997 	/* Do nothing.  This function makes this module removable. */
1998 }
1999 
2000 module_init(vsock_init_tables);
2001 module_exit(vsock_exit);
2002 
2003 MODULE_AUTHOR("VMware, Inc.");
2004 MODULE_DESCRIPTION("VMware Virtual Socket Family");
2005 MODULE_VERSION("1.0.2.0-k");
2006 MODULE_LICENSE("GPL v2");
2007