1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2a246b010SChuck Lever /*
3a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c
4a246b010SChuck Lever *
5a246b010SChuck Lever * Client-side transport implementation for sockets.
6a246b010SChuck Lever *
7113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat
8113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat
9a246b010SChuck Lever * TCP NFS related read + write fixes
10a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie>
11a246b010SChuck Lever *
12a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff.
13a246b010SChuck Lever * Fix behaviour when socket buffer is full.
14a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no>
1555aa4f58SChuck Lever *
1655aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com>
178f9d5b1aSChuck Lever *
188f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005.
198f9d5b1aSChuck Lever * <gilles.quillard@bull.net>
20a246b010SChuck Lever */
21a246b010SChuck Lever
22a246b010SChuck Lever #include <linux/types.h>
23176e21eeSChuck Lever #include <linux/string.h>
24a246b010SChuck Lever #include <linux/slab.h>
25bc25571eS\"Talpey, Thomas\ #include <linux/module.h>
26a246b010SChuck Lever #include <linux/capability.h>
27a246b010SChuck Lever #include <linux/pagemap.h>
28a246b010SChuck Lever #include <linux/errno.h>
29a246b010SChuck Lever #include <linux/socket.h>
30a246b010SChuck Lever #include <linux/in.h>
31a246b010SChuck Lever #include <linux/net.h>
32a246b010SChuck Lever #include <linux/mm.h>
33176e21eeSChuck Lever #include <linux/un.h>
34a246b010SChuck Lever #include <linux/udp.h>
35a246b010SChuck Lever #include <linux/tcp.h>
36a246b010SChuck Lever #include <linux/sunrpc/clnt.h>
375976687aSJeff Layton #include <linux/sunrpc/addr.h>
3802107148SChuck Lever #include <linux/sunrpc/sched.h>
394cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h>
4049c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h>
41a246b010SChuck Lever #include <linux/file.h>
429e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL
4344b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h>
4444b98efdSRicardo Labiaga #endif
45a246b010SChuck Lever
46a246b010SChuck Lever #include <net/sock.h>
47a246b010SChuck Lever #include <net/checksum.h>
48a246b010SChuck Lever #include <net/udp.h>
49a246b010SChuck Lever #include <net/tcp.h>
506a7eccefSChuck Lever #include <net/tls_prot.h>
5175eb6af7SChuck Lever #include <net/handshake.h>
52dea034b9SChuck Lever
53277e4ab7STrond Myklebust #include <linux/bvec.h>
546a829eb8STrond Myklebust #include <linux/highmem.h>
55277e4ab7STrond Myklebust #include <linux/uio.h>
56a1231fdaSTrond Myklebust #include <linux/sched/mm.h>
57a246b010SChuck Lever
5840e0b090SPeilin Ye #include <trace/events/sock.h>
5940b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h>
6040b5ea0cSTrond Myklebust
619e55eef4SChuck Lever #include "socklib.h"
624cfc7e60SRahul Iyer #include "sunrpc.h"
63176e21eeSChuck Lever
64176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt);
653b21f757STrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock);
667196dbb0STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
677196dbb0STrond Myklebust struct socket *sock);
68176e21eeSChuck Lever
699903cd1cSChuck Lever /*
70c556b754SChuck Lever * xprtsock tunables
71c556b754SChuck Lever */
7209acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE;
7309acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE;
7409acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE;
75c556b754SChuck Lever
7609acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT;
7709acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT;
78c556b754SChuck Lever
797d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ)
8025fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO;
817d1e8255STrond Myklebust
82c556b754SChuck Lever /*
83fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by
84c946cb69SLuis Chamberlain * calling register_sysctl() again. The files in that
85fbf76683SChuck Lever * directory become the union of all files registered there.
86fbf76683SChuck Lever *
87fbf76683SChuck Lever * We simply need to make sure that we don't collide with
88fbf76683SChuck Lever * someone else's file names!
89fbf76683SChuck Lever */
90fbf76683SChuck Lever
91fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE;
92fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE;
93d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT;
94fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT;
95fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT;
96fbf76683SChuck Lever
97fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header;
98fbf76683SChuck Lever
99d3abc739SOlga Kornievskaia static struct xprt_class xs_local_transport;
100d3abc739SOlga Kornievskaia static struct xprt_class xs_udp_transport;
101d3abc739SOlga Kornievskaia static struct xprt_class xs_tcp_transport;
10275eb6af7SChuck Lever static struct xprt_class xs_tcp_tls_transport;
103d3abc739SOlga Kornievskaia static struct xprt_class xs_bc_tcp_transport;
104d3abc739SOlga Kornievskaia
105fbf76683SChuck Lever /*
106fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP
107fbf76683SChuck Lever * socket buffers for existing UDP transports
108fbf76683SChuck Lever */
109fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = {
110fbf76683SChuck Lever {
111fbf76683SChuck Lever .procname = "udp_slot_table_entries",
112fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries,
113fbf76683SChuck Lever .maxlen = sizeof(unsigned int),
114fbf76683SChuck Lever .mode = 0644,
1156d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax,
116fbf76683SChuck Lever .extra1 = &min_slot_table_size,
117fbf76683SChuck Lever .extra2 = &max_slot_table_size
118fbf76683SChuck Lever },
119fbf76683SChuck Lever {
120fbf76683SChuck Lever .procname = "tcp_slot_table_entries",
121fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries,
122fbf76683SChuck Lever .maxlen = sizeof(unsigned int),
123fbf76683SChuck Lever .mode = 0644,
1246d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax,
125fbf76683SChuck Lever .extra1 = &min_slot_table_size,
126fbf76683SChuck Lever .extra2 = &max_slot_table_size
127fbf76683SChuck Lever },
128fbf76683SChuck Lever {
129d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries",
130d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries,
131d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int),
132d9ba131dSTrond Myklebust .mode = 0644,
133d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax,
134d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size,
135d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit
136d9ba131dSTrond Myklebust },
137d9ba131dSTrond Myklebust {
138fbf76683SChuck Lever .procname = "min_resvport",
139fbf76683SChuck Lever .data = &xprt_min_resvport,
140fbf76683SChuck Lever .maxlen = sizeof(unsigned int),
141fbf76683SChuck Lever .mode = 0644,
1426d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax,
143fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit,
144826799e6SJ. Bruce Fields .extra2 = &xprt_max_resvport_limit
145fbf76683SChuck Lever },
146fbf76683SChuck Lever {
147fbf76683SChuck Lever .procname = "max_resvport",
148fbf76683SChuck Lever .data = &xprt_max_resvport,
149fbf76683SChuck Lever .maxlen = sizeof(unsigned int),
150fbf76683SChuck Lever .mode = 0644,
1516d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax,
152826799e6SJ. Bruce Fields .extra1 = &xprt_min_resvport_limit,
153fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit
154fbf76683SChuck Lever },
155fbf76683SChuck Lever {
15625fe6142STrond Myklebust .procname = "tcp_fin_timeout",
15725fe6142STrond Myklebust .data = &xs_tcp_fin_timeout,
15825fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout),
15925fe6142STrond Myklebust .mode = 0644,
1606d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies,
16125fe6142STrond Myklebust },
162f8572d8fSEric W. Biederman { },
163fbf76683SChuck Lever };
164fbf76683SChuck Lever
165fbf76683SChuck Lever /*
16603bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper.
16703bf4b70SChuck Lever */
16803bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ)
16903bf4b70SChuck Lever
17003bf4b70SChuck Lever /*
17103bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some
17203bf4b70SChuck Lever * kind of resource problem on the local host.
17303bf4b70SChuck Lever */
17403bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ)
17503bf4b70SChuck Lever
17603bf4b70SChuck Lever /*
17703bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting
17803bf4b70SChuck Lever * to reconnect to a server that just dropped our connection.
17903bf4b70SChuck Lever *
18003bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP
18103bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP
18203bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and
18303bf4b70SChuck Lever * increase over time if the server is down or not responding.
18403bf4b70SChuck Lever */
18503bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ)
18603bf4b70SChuck Lever
18703bf4b70SChuck Lever /*
18803bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle
18903bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent
19003bf4b70SChuck Lever * holding port numbers when there is no RPC traffic.
19103bf4b70SChuck Lever */
19203bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ)
19303bf4b70SChuck Lever
19475eb6af7SChuck Lever /*
19575eb6af7SChuck Lever * TLS handshake timeout.
19675eb6af7SChuck Lever */
19775eb6af7SChuck Lever #define XS_TLS_HANDSHAKE_TO (10U * HZ)
19875eb6af7SChuck Lever
199f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
200a246b010SChuck Lever # undef RPC_DEBUG_DATA
2019903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS
202a246b010SChuck Lever #endif
203a246b010SChuck Lever
204a246b010SChuck Lever #ifdef RPC_DEBUG_DATA
xs_pktdump(char * msg,u32 * packet,unsigned int count)2059903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count)
206a246b010SChuck Lever {
207a246b010SChuck Lever u8 *buf = (u8 *) packet;
208a246b010SChuck Lever int j;
209a246b010SChuck Lever
210a246b010SChuck Lever dprintk("RPC: %s\n", msg);
211a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) {
212a246b010SChuck Lever if (!(j & 31)) {
213a246b010SChuck Lever if (j)
214a246b010SChuck Lever dprintk("\n");
215a246b010SChuck Lever dprintk("0x%04x ", j);
216a246b010SChuck Lever }
217a246b010SChuck Lever dprintk("%02x%02x%02x%02x ",
218a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]);
219a246b010SChuck Lever }
220a246b010SChuck Lever dprintk("\n");
221a246b010SChuck Lever }
222a246b010SChuck Lever #else
xs_pktdump(char * msg,u32 * packet,unsigned int count)2239903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count)
224a246b010SChuck Lever {
225a246b010SChuck Lever /* NOP */
226a246b010SChuck Lever }
227a246b010SChuck Lever #endif
228a246b010SChuck Lever
xprt_from_sock(struct sock * sk)2292118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk)
2302118071dSTrond Myklebust {
2312118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data;
2322118071dSTrond Myklebust }
2332118071dSTrond Myklebust
xs_addr(struct rpc_xprt * xprt)23495392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt)
235edb267a6SChuck Lever {
23695392c59SChuck Lever return (struct sockaddr *) &xprt->addr;
23795392c59SChuck Lever }
23895392c59SChuck Lever
xs_addr_un(struct rpc_xprt * xprt)239176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt)
240176e21eeSChuck Lever {
241176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr;
242176e21eeSChuck Lever }
243176e21eeSChuck Lever
xs_addr_in(struct rpc_xprt * xprt)24495392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt)
24595392c59SChuck Lever {
24695392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr;
24795392c59SChuck Lever }
24895392c59SChuck Lever
xs_addr_in6(struct rpc_xprt * xprt)24995392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt)
25095392c59SChuck Lever {
25195392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr;
25295392c59SChuck Lever }
25395392c59SChuck Lever
xs_format_common_peer_addresses(struct rpc_xprt * xprt)254c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt)
255c877b849SChuck Lever {
256c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt);
2579dc3b095SChuck Lever struct sockaddr_in6 *sin6;
2589dc3b095SChuck Lever struct sockaddr_in *sin;
259176e21eeSChuck Lever struct sockaddr_un *sun;
260c877b849SChuck Lever char buf[128];
261c877b849SChuck Lever
2629dc3b095SChuck Lever switch (sap->sa_family) {
263176e21eeSChuck Lever case AF_LOCAL:
264176e21eeSChuck Lever sun = xs_addr_un(xprt);
2654388ce05SNeilBrown if (sun->sun_path[0]) {
26615bcdc92SWolfram Sang strscpy(buf, sun->sun_path, sizeof(buf));
2674388ce05SNeilBrown } else {
2684388ce05SNeilBrown buf[0] = '@';
2694388ce05SNeilBrown strscpy(buf+1, sun->sun_path+1, sizeof(buf)-1);
2704388ce05SNeilBrown }
271176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] =
272176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL);
273176e21eeSChuck Lever break;
2749dc3b095SChuck Lever case AF_INET:
275176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf));
276176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] =
277176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL);
2789dc3b095SChuck Lever sin = xs_addr_in(xprt);
279fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));
2809dc3b095SChuck Lever break;
2819dc3b095SChuck Lever case AF_INET6:
282176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf));
283176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] =
284176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL);
2859dc3b095SChuck Lever sin6 = xs_addr_in6(xprt);
286fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);
2879dc3b095SChuck Lever break;
2889dc3b095SChuck Lever default:
2899dc3b095SChuck Lever BUG();
2909dc3b095SChuck Lever }
291176e21eeSChuck Lever
2929dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);
2939dc3b095SChuck Lever }
2949dc3b095SChuck Lever
xs_format_common_peer_ports(struct rpc_xprt * xprt)2959dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt)
2969dc3b095SChuck Lever {
2979dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt);
2989dc3b095SChuck Lever char buf[128];
2999dc3b095SChuck Lever
30081160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap));
301c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);
302c877b849SChuck Lever
30381160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap));
304c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);
305c877b849SChuck Lever }
306c877b849SChuck Lever
xs_format_peer_addresses(struct rpc_xprt * xprt,const char * protocol,const char * netid)3079dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt,
308b454ae90SChuck Lever const char *protocol,
309b454ae90SChuck Lever const char *netid)
310edb267a6SChuck Lever {
311b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol;
312b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid;
313c877b849SChuck Lever xs_format_common_peer_addresses(xprt);
3149dc3b095SChuck Lever xs_format_common_peer_ports(xprt);
315edb267a6SChuck Lever }
316edb267a6SChuck Lever
xs_update_peer_port(struct rpc_xprt * xprt)3179dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt)
3184b6473fbSChuck Lever {
3199dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);
3209dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]);
3214b6473fbSChuck Lever
3229dc3b095SChuck Lever xs_format_common_peer_ports(xprt);
323edb267a6SChuck Lever }
324edb267a6SChuck Lever
xs_free_peer_addresses(struct rpc_xprt * xprt)325edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt)
326edb267a6SChuck Lever {
32733e01dc7SChuck Lever unsigned int i;
32833e01dc7SChuck Lever
32933e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++)
33033e01dc7SChuck Lever switch (i) {
33133e01dc7SChuck Lever case RPC_DISPLAY_PROTO:
33233e01dc7SChuck Lever case RPC_DISPLAY_NETID:
33333e01dc7SChuck Lever continue;
33433e01dc7SChuck Lever default:
33533e01dc7SChuck Lever kfree(xprt->address_strings[i]);
33633e01dc7SChuck Lever }
337edb267a6SChuck Lever }
338edb267a6SChuck Lever
339277e4ab7STrond Myklebust static size_t
xs_alloc_sparse_pages(struct xdr_buf * buf,size_t want,gfp_t gfp)340277e4ab7STrond Myklebust xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp)
341277e4ab7STrond Myklebust {
342277e4ab7STrond Myklebust size_t i,n;
343277e4ab7STrond Myklebust
34416e5e90fSTrond Myklebust if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES))
345277e4ab7STrond Myklebust return want;
346277e4ab7STrond Myklebust n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT;
347277e4ab7STrond Myklebust for (i = 0; i < n; i++) {
348277e4ab7STrond Myklebust if (buf->pages[i])
349277e4ab7STrond Myklebust continue;
350277e4ab7STrond Myklebust buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp);
351277e4ab7STrond Myklebust if (!buf->pages[i]) {
35216e5e90fSTrond Myklebust i *= PAGE_SIZE;
35316e5e90fSTrond Myklebust return i > buf->page_base ? i - buf->page_base : 0;
354277e4ab7STrond Myklebust }
355277e4ab7STrond Myklebust }
356277e4ab7STrond Myklebust return want;
357277e4ab7STrond Myklebust }
358277e4ab7STrond Myklebust
359dea034b9SChuck Lever static int
xs_sock_process_cmsg(struct socket * sock,struct msghdr * msg,struct cmsghdr * cmsg,int ret)360dea034b9SChuck Lever xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg,
361dea034b9SChuck Lever struct cmsghdr *cmsg, int ret)
362dea034b9SChuck Lever {
36339067ddaSChuck Lever u8 content_type = tls_get_record_type(sock->sk, cmsg);
36439067ddaSChuck Lever u8 level, description;
365dea034b9SChuck Lever
366dea034b9SChuck Lever switch (content_type) {
36739067ddaSChuck Lever case 0:
36839067ddaSChuck Lever break;
369dea034b9SChuck Lever case TLS_RECORD_TYPE_DATA:
370dea034b9SChuck Lever /* TLS sets EOR at the end of each application data
371dea034b9SChuck Lever * record, even though there might be more frames
372dea034b9SChuck Lever * waiting to be decrypted.
373dea034b9SChuck Lever */
374dea034b9SChuck Lever msg->msg_flags &= ~MSG_EOR;
375dea034b9SChuck Lever break;
376dea034b9SChuck Lever case TLS_RECORD_TYPE_ALERT:
37739067ddaSChuck Lever tls_alert_recv(sock->sk, msg, &level, &description);
37839067ddaSChuck Lever ret = (level == TLS_ALERT_LEVEL_FATAL) ?
37939067ddaSChuck Lever -EACCES : -EAGAIN;
380dea034b9SChuck Lever break;
381dea034b9SChuck Lever default:
38239067ddaSChuck Lever /* discard this record type */
383dea034b9SChuck Lever ret = -EAGAIN;
384dea034b9SChuck Lever }
385dea034b9SChuck Lever return ret;
386dea034b9SChuck Lever }
387dea034b9SChuck Lever
388dea034b9SChuck Lever static int
xs_sock_recv_cmsg(struct socket * sock,struct msghdr * msg,int flags)389dea034b9SChuck Lever xs_sock_recv_cmsg(struct socket *sock, struct msghdr *msg, int flags)
390dea034b9SChuck Lever {
391dea034b9SChuck Lever union {
392dea034b9SChuck Lever struct cmsghdr cmsg;
393dea034b9SChuck Lever u8 buf[CMSG_SPACE(sizeof(u8))];
394dea034b9SChuck Lever } u;
395dea034b9SChuck Lever int ret;
396dea034b9SChuck Lever
397dea034b9SChuck Lever msg->msg_control = &u;
398dea034b9SChuck Lever msg->msg_controllen = sizeof(u);
399dea034b9SChuck Lever ret = sock_recvmsg(sock, msg, flags);
400dea034b9SChuck Lever if (msg->msg_controllen != sizeof(u))
401dea034b9SChuck Lever ret = xs_sock_process_cmsg(sock, msg, &u.cmsg, ret);
402dea034b9SChuck Lever return ret;
403dea034b9SChuck Lever }
404dea034b9SChuck Lever
405277e4ab7STrond Myklebust static ssize_t
xs_sock_recvmsg(struct socket * sock,struct msghdr * msg,int flags,size_t seek)406277e4ab7STrond Myklebust xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek)
407277e4ab7STrond Myklebust {
408277e4ab7STrond Myklebust ssize_t ret;
409277e4ab7STrond Myklebust if (seek != 0)
410277e4ab7STrond Myklebust iov_iter_advance(&msg->msg_iter, seek);
411dea034b9SChuck Lever ret = xs_sock_recv_cmsg(sock, msg, flags);
412277e4ab7STrond Myklebust return ret > 0 ? ret + seek : ret;
413277e4ab7STrond Myklebust }
414277e4ab7STrond Myklebust
415277e4ab7STrond Myklebust static ssize_t
xs_read_kvec(struct socket * sock,struct msghdr * msg,int flags,struct kvec * kvec,size_t count,size_t seek)416277e4ab7STrond Myklebust xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags,
417277e4ab7STrond Myklebust struct kvec *kvec, size_t count, size_t seek)
418277e4ab7STrond Myklebust {
419de4eda9dSAl Viro iov_iter_kvec(&msg->msg_iter, ITER_DEST, kvec, 1, count);
420277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek);
421277e4ab7STrond Myklebust }
422277e4ab7STrond Myklebust
423277e4ab7STrond Myklebust static ssize_t
xs_read_bvec(struct socket * sock,struct msghdr * msg,int flags,struct bio_vec * bvec,unsigned long nr,size_t count,size_t seek)424277e4ab7STrond Myklebust xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags,
425277e4ab7STrond Myklebust struct bio_vec *bvec, unsigned long nr, size_t count,
426277e4ab7STrond Myklebust size_t seek)
427277e4ab7STrond Myklebust {
428de4eda9dSAl Viro iov_iter_bvec(&msg->msg_iter, ITER_DEST, bvec, nr, count);
429277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek);
430277e4ab7STrond Myklebust }
431277e4ab7STrond Myklebust
432277e4ab7STrond Myklebust static ssize_t
xs_read_discard(struct socket * sock,struct msghdr * msg,int flags,size_t count)433277e4ab7STrond Myklebust xs_read_discard(struct socket *sock, struct msghdr *msg, int flags,
434277e4ab7STrond Myklebust size_t count)
435277e4ab7STrond Myklebust {
436de4eda9dSAl Viro iov_iter_discard(&msg->msg_iter, ITER_DEST, count);
437dea034b9SChuck Lever return xs_sock_recv_cmsg(sock, msg, flags);
438277e4ab7STrond Myklebust }
439277e4ab7STrond Myklebust
4406a829eb8STrond Myklebust #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
4416a829eb8STrond Myklebust static void
xs_flush_bvec(const struct bio_vec * bvec,size_t count,size_t seek)4426a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
4436a829eb8STrond Myklebust {
4446a829eb8STrond Myklebust struct bvec_iter bi = {
4456a829eb8STrond Myklebust .bi_size = count,
4466a829eb8STrond Myklebust };
4476a829eb8STrond Myklebust struct bio_vec bv;
4486a829eb8STrond Myklebust
4496a829eb8STrond Myklebust bvec_iter_advance(bvec, &bi, seek & PAGE_MASK);
4506a829eb8STrond Myklebust for_each_bvec(bv, bvec, bi, bi)
4516a829eb8STrond Myklebust flush_dcache_page(bv.bv_page);
4526a829eb8STrond Myklebust }
4536a829eb8STrond Myklebust #else
4546a829eb8STrond Myklebust static inline void
xs_flush_bvec(const struct bio_vec * bvec,size_t count,size_t seek)4556a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek)
4566a829eb8STrond Myklebust {
4576a829eb8STrond Myklebust }
4586a829eb8STrond Myklebust #endif
4596a829eb8STrond Myklebust
460277e4ab7STrond Myklebust static ssize_t
xs_read_xdr_buf(struct socket * sock,struct msghdr * msg,int flags,struct xdr_buf * buf,size_t count,size_t seek,size_t * read)461277e4ab7STrond Myklebust xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags,
462277e4ab7STrond Myklebust struct xdr_buf *buf, size_t count, size_t seek, size_t *read)
463277e4ab7STrond Myklebust {
464277e4ab7STrond Myklebust size_t want, seek_init = seek, offset = 0;
465277e4ab7STrond Myklebust ssize_t ret;
466277e4ab7STrond Myklebust
467277e4ab7STrond Myklebust want = min_t(size_t, count, buf->head[0].iov_len);
468e92053a5STrond Myklebust if (seek < want) {
469277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek);
470277e4ab7STrond Myklebust if (ret <= 0)
471277e4ab7STrond Myklebust goto sock_err;
472277e4ab7STrond Myklebust offset += ret;
473277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
474277e4ab7STrond Myklebust goto out;
475277e4ab7STrond Myklebust if (ret != want)
476c4433055STrond Myklebust goto out;
477277e4ab7STrond Myklebust seek = 0;
478277e4ab7STrond Myklebust } else {
479e92053a5STrond Myklebust seek -= want;
480e92053a5STrond Myklebust offset += want;
481277e4ab7STrond Myklebust }
48216e5e90fSTrond Myklebust
483b2648015STrond Myklebust want = xs_alloc_sparse_pages(
484b2648015STrond Myklebust buf, min_t(size_t, count - offset, buf->page_len),
485b2648015STrond Myklebust GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
48616e5e90fSTrond Myklebust if (seek < want) {
487277e4ab7STrond Myklebust ret = xs_read_bvec(sock, msg, flags, buf->bvec,
488277e4ab7STrond Myklebust xdr_buf_pagecount(buf),
489277e4ab7STrond Myklebust want + buf->page_base,
490277e4ab7STrond Myklebust seek + buf->page_base);
491277e4ab7STrond Myklebust if (ret <= 0)
492277e4ab7STrond Myklebust goto sock_err;
4936a829eb8STrond Myklebust xs_flush_bvec(buf->bvec, ret, seek + buf->page_base);
494ac9645c8SDan Aloni ret -= buf->page_base;
495ac9645c8SDan Aloni offset += ret;
496277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
497277e4ab7STrond Myklebust goto out;
498277e4ab7STrond Myklebust if (ret != want)
499c4433055STrond Myklebust goto out;
500277e4ab7STrond Myklebust seek = 0;
501277e4ab7STrond Myklebust } else {
50216e5e90fSTrond Myklebust seek -= want;
50316e5e90fSTrond Myklebust offset += want;
504277e4ab7STrond Myklebust }
50516e5e90fSTrond Myklebust
506277e4ab7STrond Myklebust want = min_t(size_t, count - offset, buf->tail[0].iov_len);
507e92053a5STrond Myklebust if (seek < want) {
508277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek);
509277e4ab7STrond Myklebust if (ret <= 0)
510277e4ab7STrond Myklebust goto sock_err;
511277e4ab7STrond Myklebust offset += ret;
512277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC))
513277e4ab7STrond Myklebust goto out;
514277e4ab7STrond Myklebust if (ret != want)
515c4433055STrond Myklebust goto out;
5169734ad57STrond Myklebust } else if (offset < seek_init)
517e92053a5STrond Myklebust offset = seek_init;
518277e4ab7STrond Myklebust ret = -EMSGSIZE;
519277e4ab7STrond Myklebust out:
520277e4ab7STrond Myklebust *read = offset - seek_init;
521277e4ab7STrond Myklebust return ret;
522277e4ab7STrond Myklebust sock_err:
523277e4ab7STrond Myklebust offset += seek;
524277e4ab7STrond Myklebust goto out;
525277e4ab7STrond Myklebust }
526277e4ab7STrond Myklebust
527277e4ab7STrond Myklebust static void
xs_read_header(struct sock_xprt * transport,struct xdr_buf * buf)528277e4ab7STrond Myklebust xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf)
529277e4ab7STrond Myklebust {
530277e4ab7STrond Myklebust if (!transport->recv.copied) {
531277e4ab7STrond Myklebust if (buf->head[0].iov_len >= transport->recv.offset)
532277e4ab7STrond Myklebust memcpy(buf->head[0].iov_base,
533277e4ab7STrond Myklebust &transport->recv.xid,
534277e4ab7STrond Myklebust transport->recv.offset);
535277e4ab7STrond Myklebust transport->recv.copied = transport->recv.offset;
536277e4ab7STrond Myklebust }
537277e4ab7STrond Myklebust }
538277e4ab7STrond Myklebust
539277e4ab7STrond Myklebust static bool
xs_read_stream_request_done(struct sock_xprt * transport)540277e4ab7STrond Myklebust xs_read_stream_request_done(struct sock_xprt *transport)
541277e4ab7STrond Myklebust {
542277e4ab7STrond Myklebust return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT);
543277e4ab7STrond Myklebust }
544277e4ab7STrond Myklebust
545e92053a5STrond Myklebust static void
xs_read_stream_check_eor(struct sock_xprt * transport,struct msghdr * msg)546e92053a5STrond Myklebust xs_read_stream_check_eor(struct sock_xprt *transport,
547e92053a5STrond Myklebust struct msghdr *msg)
548e92053a5STrond Myklebust {
549e92053a5STrond Myklebust if (xs_read_stream_request_done(transport))
550e92053a5STrond Myklebust msg->msg_flags |= MSG_EOR;
551e92053a5STrond Myklebust }
552e92053a5STrond Myklebust
553277e4ab7STrond Myklebust static ssize_t
xs_read_stream_request(struct sock_xprt * transport,struct msghdr * msg,int flags,struct rpc_rqst * req)554277e4ab7STrond Myklebust xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg,
555277e4ab7STrond Myklebust int flags, struct rpc_rqst *req)
556277e4ab7STrond Myklebust {
557277e4ab7STrond Myklebust struct xdr_buf *buf = &req->rq_private_buf;
5583f649ab7SKees Cook size_t want, read;
5593f649ab7SKees Cook ssize_t ret;
560277e4ab7STrond Myklebust
561277e4ab7STrond Myklebust xs_read_header(transport, buf);
562277e4ab7STrond Myklebust
563277e4ab7STrond Myklebust want = transport->recv.len - transport->recv.offset;
564e92053a5STrond Myklebust if (want != 0) {
565277e4ab7STrond Myklebust ret = xs_read_xdr_buf(transport->sock, msg, flags, buf,
566e92053a5STrond Myklebust transport->recv.copied + want,
567e92053a5STrond Myklebust transport->recv.copied,
568277e4ab7STrond Myklebust &read);
569277e4ab7STrond Myklebust transport->recv.offset += read;
570277e4ab7STrond Myklebust transport->recv.copied += read;
571277e4ab7STrond Myklebust }
572277e4ab7STrond Myklebust
573727fcc64STrond Myklebust if (transport->recv.offset == transport->recv.len)
574727fcc64STrond Myklebust xs_read_stream_check_eor(transport, msg);
575727fcc64STrond Myklebust
576e92053a5STrond Myklebust if (want == 0)
577e92053a5STrond Myklebust return 0;
578e92053a5STrond Myklebust
579277e4ab7STrond Myklebust switch (ret) {
580c4433055STrond Myklebust default:
581c4433055STrond Myklebust break;
58226781eabSTrond Myklebust case -EFAULT:
583277e4ab7STrond Myklebust case -EMSGSIZE:
58426781eabSTrond Myklebust msg->msg_flags |= MSG_TRUNC;
585c4433055STrond Myklebust return read;
586277e4ab7STrond Myklebust case 0:
587277e4ab7STrond Myklebust return -ESHUTDOWN;
588277e4ab7STrond Myklebust }
589c4433055STrond Myklebust return ret < 0 ? ret : read;
590277e4ab7STrond Myklebust }
591277e4ab7STrond Myklebust
592277e4ab7STrond Myklebust static size_t
xs_read_stream_headersize(bool isfrag)593277e4ab7STrond Myklebust xs_read_stream_headersize(bool isfrag)
594277e4ab7STrond Myklebust {
595277e4ab7STrond Myklebust if (isfrag)
596277e4ab7STrond Myklebust return sizeof(__be32);
597277e4ab7STrond Myklebust return 3 * sizeof(__be32);
598277e4ab7STrond Myklebust }
599277e4ab7STrond Myklebust
600277e4ab7STrond Myklebust static ssize_t
xs_read_stream_header(struct sock_xprt * transport,struct msghdr * msg,int flags,size_t want,size_t seek)601277e4ab7STrond Myklebust xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg,
602277e4ab7STrond Myklebust int flags, size_t want, size_t seek)
603277e4ab7STrond Myklebust {
604277e4ab7STrond Myklebust struct kvec kvec = {
605277e4ab7STrond Myklebust .iov_base = &transport->recv.fraghdr,
606277e4ab7STrond Myklebust .iov_len = want,
607277e4ab7STrond Myklebust };
608277e4ab7STrond Myklebust return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek);
609277e4ab7STrond Myklebust }
610277e4ab7STrond Myklebust
611277e4ab7STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL)
612277e4ab7STrond Myklebust static ssize_t
xs_read_stream_call(struct sock_xprt * transport,struct msghdr * msg,int flags)613277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
614277e4ab7STrond Myklebust {
615277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt;
616277e4ab7STrond Myklebust struct rpc_rqst *req;
617277e4ab7STrond Myklebust ssize_t ret;
618277e4ab7STrond Myklebust
61998b5cee3SBenjamin Coddington /* Is this transport associated with the backchannel? */
62098b5cee3SBenjamin Coddington if (!xprt->bc_serv)
62198b5cee3SBenjamin Coddington return -ESHUTDOWN;
62298b5cee3SBenjamin Coddington
623277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */
624277e4ab7STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->recv.xid);
625277e4ab7STrond Myklebust if (!req) {
626277e4ab7STrond Myklebust printk(KERN_WARNING "Callback slot table overflowed\n");
627277e4ab7STrond Myklebust return -ESHUTDOWN;
628277e4ab7STrond Myklebust }
62945835a63STrond Myklebust if (transport->recv.copied && !req->rq_private_buf.len)
63045835a63STrond Myklebust return -ESHUTDOWN;
631277e4ab7STrond Myklebust
632277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req);
633277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
634c4433055STrond Myklebust xprt_complete_bc_request(req, transport->recv.copied);
63545835a63STrond Myklebust else
63645835a63STrond Myklebust req->rq_private_buf.len = transport->recv.copied;
637277e4ab7STrond Myklebust
638277e4ab7STrond Myklebust return ret;
639277e4ab7STrond Myklebust }
640277e4ab7STrond Myklebust #else /* CONFIG_SUNRPC_BACKCHANNEL */
641277e4ab7STrond Myklebust static ssize_t
xs_read_stream_call(struct sock_xprt * transport,struct msghdr * msg,int flags)642277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags)
643277e4ab7STrond Myklebust {
644277e4ab7STrond Myklebust return -ESHUTDOWN;
645277e4ab7STrond Myklebust }
646277e4ab7STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */
647277e4ab7STrond Myklebust
648277e4ab7STrond Myklebust static ssize_t
xs_read_stream_reply(struct sock_xprt * transport,struct msghdr * msg,int flags)649277e4ab7STrond Myklebust xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags)
650277e4ab7STrond Myklebust {
651277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt;
652277e4ab7STrond Myklebust struct rpc_rqst *req;
653277e4ab7STrond Myklebust ssize_t ret = 0;
654277e4ab7STrond Myklebust
655277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */
656277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock);
657277e4ab7STrond Myklebust req = xprt_lookup_rqst(xprt, transport->recv.xid);
65845835a63STrond Myklebust if (!req || (transport->recv.copied && !req->rq_private_buf.len)) {
659277e4ab7STrond Myklebust msg->msg_flags |= MSG_TRUNC;
660277e4ab7STrond Myklebust goto out;
661277e4ab7STrond Myklebust }
662277e4ab7STrond Myklebust xprt_pin_rqst(req);
663277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock);
664277e4ab7STrond Myklebust
665277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req);
666277e4ab7STrond Myklebust
667277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock);
668277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC))
669c4433055STrond Myklebust xprt_complete_rqst(req->rq_task, transport->recv.copied);
67045835a63STrond Myklebust else
67145835a63STrond Myklebust req->rq_private_buf.len = transport->recv.copied;
672277e4ab7STrond Myklebust xprt_unpin_rqst(req);
673277e4ab7STrond Myklebust out:
674277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock);
675277e4ab7STrond Myklebust return ret;
676277e4ab7STrond Myklebust }
677277e4ab7STrond Myklebust
678277e4ab7STrond Myklebust static ssize_t
xs_read_stream(struct sock_xprt * transport,int flags)679277e4ab7STrond Myklebust xs_read_stream(struct sock_xprt *transport, int flags)
680277e4ab7STrond Myklebust {
681277e4ab7STrond Myklebust struct msghdr msg = { 0 };
682277e4ab7STrond Myklebust size_t want, read = 0;
683277e4ab7STrond Myklebust ssize_t ret = 0;
684277e4ab7STrond Myklebust
685277e4ab7STrond Myklebust if (transport->recv.len == 0) {
686277e4ab7STrond Myklebust want = xs_read_stream_headersize(transport->recv.copied != 0);
687277e4ab7STrond Myklebust ret = xs_read_stream_header(transport, &msg, flags, want,
688277e4ab7STrond Myklebust transport->recv.offset);
689277e4ab7STrond Myklebust if (ret <= 0)
690277e4ab7STrond Myklebust goto out_err;
691277e4ab7STrond Myklebust transport->recv.offset = ret;
692c4433055STrond Myklebust if (transport->recv.offset != want)
693c4433055STrond Myklebust return transport->recv.offset;
694277e4ab7STrond Myklebust transport->recv.len = be32_to_cpu(transport->recv.fraghdr) &
695277e4ab7STrond Myklebust RPC_FRAGMENT_SIZE_MASK;
696277e4ab7STrond Myklebust transport->recv.offset -= sizeof(transport->recv.fraghdr);
697277e4ab7STrond Myklebust read = ret;
698277e4ab7STrond Myklebust }
699277e4ab7STrond Myklebust
700277e4ab7STrond Myklebust switch (be32_to_cpu(transport->recv.calldir)) {
701c4433055STrond Myklebust default:
702c4433055STrond Myklebust msg.msg_flags |= MSG_TRUNC;
703c4433055STrond Myklebust break;
704277e4ab7STrond Myklebust case RPC_CALL:
705277e4ab7STrond Myklebust ret = xs_read_stream_call(transport, &msg, flags);
706277e4ab7STrond Myklebust break;
707277e4ab7STrond Myklebust case RPC_REPLY:
708277e4ab7STrond Myklebust ret = xs_read_stream_reply(transport, &msg, flags);
709277e4ab7STrond Myklebust }
710277e4ab7STrond Myklebust if (msg.msg_flags & MSG_TRUNC) {
711277e4ab7STrond Myklebust transport->recv.calldir = cpu_to_be32(-1);
712277e4ab7STrond Myklebust transport->recv.copied = -1;
713277e4ab7STrond Myklebust }
714277e4ab7STrond Myklebust if (ret < 0)
715277e4ab7STrond Myklebust goto out_err;
716277e4ab7STrond Myklebust read += ret;
717277e4ab7STrond Myklebust if (transport->recv.offset < transport->recv.len) {
718c4433055STrond Myklebust if (!(msg.msg_flags & MSG_TRUNC))
719c4433055STrond Myklebust return read;
720b76a5afdSTrond Myklebust msg.msg_flags = 0;
721277e4ab7STrond Myklebust ret = xs_read_discard(transport->sock, &msg, flags,
722277e4ab7STrond Myklebust transport->recv.len - transport->recv.offset);
723277e4ab7STrond Myklebust if (ret <= 0)
724277e4ab7STrond Myklebust goto out_err;
725277e4ab7STrond Myklebust transport->recv.offset += ret;
726277e4ab7STrond Myklebust read += ret;
727277e4ab7STrond Myklebust if (transport->recv.offset != transport->recv.len)
728c4433055STrond Myklebust return read;
729277e4ab7STrond Myklebust }
730277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) {
731c50b8ee0STrond Myklebust trace_xs_stream_read_request(transport);
732277e4ab7STrond Myklebust transport->recv.copied = 0;
733277e4ab7STrond Myklebust }
734277e4ab7STrond Myklebust transport->recv.offset = 0;
735277e4ab7STrond Myklebust transport->recv.len = 0;
736277e4ab7STrond Myklebust return read;
737277e4ab7STrond Myklebust out_err:
73879462857STrond Myklebust return ret != 0 ? ret : -ESHUTDOWN;
739277e4ab7STrond Myklebust }
740277e4ab7STrond Myklebust
xs_poll_socket(struct sock_xprt * transport)7410ffe86f4STrond Myklebust static __poll_t xs_poll_socket(struct sock_xprt *transport)
7420ffe86f4STrond Myklebust {
743a73881c9STrond Myklebust return transport->sock->ops->poll(transport->file, transport->sock,
744a73881c9STrond Myklebust NULL);
7450ffe86f4STrond Myklebust }
7460ffe86f4STrond Myklebust
xs_poll_socket_readable(struct sock_xprt * transport)7470ffe86f4STrond Myklebust static bool xs_poll_socket_readable(struct sock_xprt *transport)
7480ffe86f4STrond Myklebust {
7490ffe86f4STrond Myklebust __poll_t events = xs_poll_socket(transport);
7500ffe86f4STrond Myklebust
7510ffe86f4STrond Myklebust return (events & (EPOLLIN | EPOLLRDNORM)) && !(events & EPOLLRDHUP);
7520ffe86f4STrond Myklebust }
7530ffe86f4STrond Myklebust
xs_poll_check_readable(struct sock_xprt * transport)7540ffe86f4STrond Myklebust static void xs_poll_check_readable(struct sock_xprt *transport)
7550ffe86f4STrond Myklebust {
7560ffe86f4STrond Myklebust
7570ffe86f4STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
7580d3ca07fSChuck Lever if (test_bit(XPRT_SOCK_IGNORE_RECV, &transport->sock_state))
7590d3ca07fSChuck Lever return;
7600ffe86f4STrond Myklebust if (!xs_poll_socket_readable(transport))
7610ffe86f4STrond Myklebust return;
7620ffe86f4STrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
7630ffe86f4STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker);
7640ffe86f4STrond Myklebust }
7650ffe86f4STrond Myklebust
xs_stream_data_receive(struct sock_xprt * transport)766c50b8ee0STrond Myklebust static void xs_stream_data_receive(struct sock_xprt *transport)
767c50b8ee0STrond Myklebust {
768c50b8ee0STrond Myklebust size_t read = 0;
769c50b8ee0STrond Myklebust ssize_t ret = 0;
770c50b8ee0STrond Myklebust
771c50b8ee0STrond Myklebust mutex_lock(&transport->recv_mutex);
772c50b8ee0STrond Myklebust if (transport->sock == NULL)
773c50b8ee0STrond Myklebust goto out;
774c50b8ee0STrond Myklebust for (;;) {
775c50b8ee0STrond Myklebust ret = xs_read_stream(transport, MSG_DONTWAIT);
776c4433055STrond Myklebust if (ret < 0)
777c50b8ee0STrond Myklebust break;
778c50b8ee0STrond Myklebust read += ret;
779c50b8ee0STrond Myklebust cond_resched();
780c50b8ee0STrond Myklebust }
7815f52a9d4STrond Myklebust if (ret == -ESHUTDOWN)
7825f52a9d4STrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR);
78339067ddaSChuck Lever else if (ret == -EACCES)
78439067ddaSChuck Lever xprt_wake_pending_tasks(&transport->xprt, -EACCES);
7855f52a9d4STrond Myklebust else
7860ffe86f4STrond Myklebust xs_poll_check_readable(transport);
787c50b8ee0STrond Myklebust out:
788c50b8ee0STrond Myklebust mutex_unlock(&transport->recv_mutex);
789c50b8ee0STrond Myklebust trace_xs_stream_read_data(&transport->xprt, ret, read);
790c50b8ee0STrond Myklebust }
791c50b8ee0STrond Myklebust
xs_stream_data_receive_workfn(struct work_struct * work)792c50b8ee0STrond Myklebust static void xs_stream_data_receive_workfn(struct work_struct *work)
793c50b8ee0STrond Myklebust {
794c50b8ee0STrond Myklebust struct sock_xprt *transport =
795c50b8ee0STrond Myklebust container_of(work, struct sock_xprt, recv_worker);
796a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save();
797a1231fdaSTrond Myklebust
798c50b8ee0STrond Myklebust xs_stream_data_receive(transport);
799a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags);
800c50b8ee0STrond Myklebust }
801c50b8ee0STrond Myklebust
802550aebfeSTrond Myklebust static void
xs_stream_reset_connect(struct sock_xprt * transport)803550aebfeSTrond Myklebust xs_stream_reset_connect(struct sock_xprt *transport)
804550aebfeSTrond Myklebust {
805550aebfeSTrond Myklebust transport->recv.offset = 0;
806550aebfeSTrond Myklebust transport->recv.len = 0;
807550aebfeSTrond Myklebust transport->recv.copied = 0;
808550aebfeSTrond Myklebust transport->xmit.offset = 0;
809ae053551STrond Myklebust }
810ae053551STrond Myklebust
811ae053551STrond Myklebust static void
xs_stream_start_connect(struct sock_xprt * transport)812ae053551STrond Myklebust xs_stream_start_connect(struct sock_xprt *transport)
813ae053551STrond Myklebust {
814550aebfeSTrond Myklebust transport->xprt.stat.connect_count++;
815550aebfeSTrond Myklebust transport->xprt.stat.connect_start = jiffies;
816550aebfeSTrond Myklebust }
817550aebfeSTrond Myklebust
818b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
819b4b5cc85SChuck Lever
8209903cd1cSChuck Lever /**
821c544577dSTrond Myklebust * xs_nospace - handle transmit was incomplete
82250f484e2STrond Myklebust * @req: pointer to RPC request
8237496b59fSTrond Myklebust * @transport: pointer to struct sock_xprt
8249903cd1cSChuck Lever *
825a246b010SChuck Lever */
xs_nospace(struct rpc_rqst * req,struct sock_xprt * transport)8267496b59fSTrond Myklebust static int xs_nospace(struct rpc_rqst *req, struct sock_xprt *transport)
827a246b010SChuck Lever {
8287496b59fSTrond Myklebust struct rpc_xprt *xprt = &transport->xprt;
82906ea0bfeSTrond Myklebust struct sock *sk = transport->inet;
83024ca9a84STrond Myklebust int ret = -EAGAIN;
831a246b010SChuck Lever
832015747d2SChuck Lever trace_rpc_socket_nospace(req, transport);
833a246b010SChuck Lever
834262965f5SChuck Lever /* Protect against races with write_space */
835b5e92419STrond Myklebust spin_lock(&xprt->transport_lock);
836a246b010SChuck Lever
837262965f5SChuck Lever /* Don't race with disconnect */
838b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) {
8397496b59fSTrond Myklebust /* wait for more buffer space */
8402790a624STrond Myklebust set_bit(XPRT_SOCK_NOSPACE, &transport->sock_state);
8417496b59fSTrond Myklebust set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
8427496b59fSTrond Myklebust sk->sk_write_pending++;
8437496b59fSTrond Myklebust xprt_wait_for_buffer_space(xprt);
8447496b59fSTrond Myklebust } else
8457496b59fSTrond Myklebust ret = -ENOTCONN;
8467496b59fSTrond Myklebust
8477496b59fSTrond Myklebust spin_unlock(&xprt->transport_lock);
8487496b59fSTrond Myklebust return ret;
849d48f9ce7SDavid Vrabel }
8507496b59fSTrond Myklebust
xs_sock_nospace(struct rpc_rqst * req)8517496b59fSTrond Myklebust static int xs_sock_nospace(struct rpc_rqst *req)
8527496b59fSTrond Myklebust {
8537496b59fSTrond Myklebust struct sock_xprt *transport =
8547496b59fSTrond Myklebust container_of(req->rq_xprt, struct sock_xprt, xprt);
8557496b59fSTrond Myklebust struct sock *sk = transport->inet;
8567496b59fSTrond Myklebust int ret = -EAGAIN;
8577496b59fSTrond Myklebust
8587496b59fSTrond Myklebust lock_sock(sk);
8597496b59fSTrond Myklebust if (!sock_writeable(sk))
8607496b59fSTrond Myklebust ret = xs_nospace(req, transport);
8617496b59fSTrond Myklebust release_sock(sk);
8627496b59fSTrond Myklebust return ret;
8637496b59fSTrond Myklebust }
8647496b59fSTrond Myklebust
xs_stream_nospace(struct rpc_rqst * req,bool vm_wait)865d0afde5fSTrond Myklebust static int xs_stream_nospace(struct rpc_rqst *req, bool vm_wait)
8667496b59fSTrond Myklebust {
8677496b59fSTrond Myklebust struct sock_xprt *transport =
8687496b59fSTrond Myklebust container_of(req->rq_xprt, struct sock_xprt, xprt);
8697496b59fSTrond Myklebust struct sock *sk = transport->inet;
8707496b59fSTrond Myklebust int ret = -EAGAIN;
8717496b59fSTrond Myklebust
872d0afde5fSTrond Myklebust if (vm_wait)
873d0afde5fSTrond Myklebust return -ENOBUFS;
8747496b59fSTrond Myklebust lock_sock(sk);
8757496b59fSTrond Myklebust if (!sk_stream_memory_free(sk))
8767496b59fSTrond Myklebust ret = xs_nospace(req, transport);
8777496b59fSTrond Myklebust release_sock(sk);
8785e3771ceSTrond Myklebust return ret;
879a246b010SChuck Lever }
880a246b010SChuck Lever
xs_stream_prepare_request(struct rpc_rqst * req,struct xdr_buf * buf)88172691a26STrond Myklebust static int xs_stream_prepare_request(struct rpc_rqst *req, struct xdr_buf *buf)
882277e4ab7STrond Myklebust {
88372691a26STrond Myklebust return xdr_alloc_bvec(buf, rpc_task_gfp_mask());
884277e4ab7STrond Myklebust }
885277e4ab7STrond Myklebust
88661677eeeSChuck Lever /*
8874cd34e7cSTrond Myklebust * Determine if the previous message in the stream was aborted before it
8884cd34e7cSTrond Myklebust * could complete transmission.
8894cd34e7cSTrond Myklebust */
8904cd34e7cSTrond Myklebust static bool
xs_send_request_was_aborted(struct sock_xprt * transport,struct rpc_rqst * req)8914cd34e7cSTrond Myklebust xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req)
8924cd34e7cSTrond Myklebust {
8934cd34e7cSTrond Myklebust return transport->xmit.offset != 0 && req->rq_bytes_sent == 0;
8944cd34e7cSTrond Myklebust }
8954cd34e7cSTrond Myklebust
8964cd34e7cSTrond Myklebust /*
89706b5fc3aSTrond Myklebust * Return the stream record marker field for a record of length < 2^31-1
89861677eeeSChuck Lever */
89906b5fc3aSTrond Myklebust static rpc_fraghdr
xs_stream_record_marker(struct xdr_buf * xdr)90006b5fc3aSTrond Myklebust xs_stream_record_marker(struct xdr_buf *xdr)
90161677eeeSChuck Lever {
90206b5fc3aSTrond Myklebust if (!xdr->len)
90306b5fc3aSTrond Myklebust return 0;
90406b5fc3aSTrond Myklebust return cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | (u32)xdr->len);
90561677eeeSChuck Lever }
90661677eeeSChuck Lever
9079903cd1cSChuck Lever /**
908176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket
90950f484e2STrond Myklebust * @req: pointer to RPC request
910176e21eeSChuck Lever *
911176e21eeSChuck Lever * Return values:
912176e21eeSChuck Lever * 0: The request has been sent
913176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to
914176e21eeSChuck Lever * complete the request
915176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again
91612b20ce3SBhaskar Chowdhury * other: Some other error occurred, the request was not sent
917176e21eeSChuck Lever */
xs_local_send_request(struct rpc_rqst * req)918adfa7144STrond Myklebust static int xs_local_send_request(struct rpc_rqst *req)
919176e21eeSChuck Lever {
920176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt;
921176e21eeSChuck Lever struct sock_xprt *transport =
922176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt);
923176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf;
9247e3d3620STrond Myklebust rpc_fraghdr rm = xs_stream_record_marker(xdr);
9257e3d3620STrond Myklebust unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
9269e55eef4SChuck Lever struct msghdr msg = {
9279e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS,
9289e55eef4SChuck Lever };
929d0afde5fSTrond Myklebust bool vm_wait;
9303f649ab7SKees Cook unsigned int sent;
931176e21eeSChuck Lever int status;
932176e21eeSChuck Lever
9334cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */
9344cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) {
935f0043206STrond Myklebust xprt_force_disconnect(xprt);
9364cd34e7cSTrond Myklebust return -ENOTCONN;
9374cd34e7cSTrond Myklebust }
9384cd34e7cSTrond Myklebust
939176e21eeSChuck Lever xs_pktdump("packet data:",
940176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len);
941176e21eeSChuck Lever
942d0afde5fSTrond Myklebust vm_wait = sk_stream_is_writeable(transport->inet) ? true : false;
943d0afde5fSTrond Myklebust
94478215759SChuck Lever req->rq_xtime = ktime_get();
9459e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
9467e3d3620STrond Myklebust transport->xmit.offset, rm, &sent);
947176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n",
9486c7a64e5STrond Myklebust __func__, xdr->len - transport->xmit.offset, status);
949743c69e7SNeilBrown
950f279cd00SJason Baron if (likely(sent > 0) || status == 0) {
9516c7a64e5STrond Myklebust transport->xmit.offset += sent;
9526c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset;
9537e3d3620STrond Myklebust if (likely(req->rq_bytes_sent >= msglen)) {
9546c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset;
9556c7a64e5STrond Myklebust transport->xmit.offset = 0;
956176e21eeSChuck Lever return 0;
957176e21eeSChuck Lever }
958176e21eeSChuck Lever status = -EAGAIN;
959d0afde5fSTrond Myklebust vm_wait = false;
960176e21eeSChuck Lever }
961176e21eeSChuck Lever
962176e21eeSChuck Lever switch (status) {
963176e21eeSChuck Lever case -EAGAIN:
964d0afde5fSTrond Myklebust status = xs_stream_nospace(req, vm_wait);
965176e21eeSChuck Lever break;
966176e21eeSChuck Lever default:
967176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n",
968176e21eeSChuck Lever -status);
969df561f66SGustavo A. R. Silva fallthrough;
970176e21eeSChuck Lever case -EPIPE:
971f0043206STrond Myklebust xprt_force_disconnect(xprt);
972176e21eeSChuck Lever status = -ENOTCONN;
973176e21eeSChuck Lever }
974176e21eeSChuck Lever
975176e21eeSChuck Lever return status;
976176e21eeSChuck Lever }
977176e21eeSChuck Lever
978176e21eeSChuck Lever /**
979262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket
98050f484e2STrond Myklebust * @req: pointer to RPC request
9819903cd1cSChuck Lever *
9829903cd1cSChuck Lever * Return values:
9839903cd1cSChuck Lever * 0: The request has been sent
9849903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to
9859903cd1cSChuck Lever * complete the request
986262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again
98725985edcSLucas De Marchi * other: Some other error occurred, the request was not sent
9889903cd1cSChuck Lever */
xs_udp_send_request(struct rpc_rqst * req)989adfa7144STrond Myklebust static int xs_udp_send_request(struct rpc_rqst *req)
990a246b010SChuck Lever {
991a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt;
992ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
993262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf;
9949e55eef4SChuck Lever struct msghdr msg = {
9959e55eef4SChuck Lever .msg_name = xs_addr(xprt),
9969e55eef4SChuck Lever .msg_namelen = xprt->addrlen,
9979e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS,
9989e55eef4SChuck Lever };
9993f649ab7SKees Cook unsigned int sent;
1000262965f5SChuck Lever int status;
1001262965f5SChuck Lever
1002262965f5SChuck Lever xs_pktdump("packet data:",
1003262965f5SChuck Lever req->rq_svec->iov_base,
1004262965f5SChuck Lever req->rq_svec->iov_len);
1005262965f5SChuck Lever
100601d37c42STrond Myklebust if (!xprt_bound(xprt))
100701d37c42STrond Myklebust return -ENOTCONN;
100875891f50STrond Myklebust
100975891f50STrond Myklebust if (!xprt_request_get_cong(xprt, req))
101075891f50STrond Myklebust return -EBADSLT;
101175891f50STrond Myklebust
1012ff053dbbSTrond Myklebust status = xdr_alloc_bvec(xdr, rpc_task_gfp_mask());
1013ff053dbbSTrond Myklebust if (status < 0)
1014ff053dbbSTrond Myklebust return status;
101578215759SChuck Lever req->rq_xtime = ktime_get();
10169e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, 0, &sent);
1017262965f5SChuck Lever
1018262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n",
10196c7a64e5STrond Myklebust xdr->len, status);
1020262965f5SChuck Lever
10213dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */
10223dedbb5cSJason Baron if (status == -EPERM)
10233dedbb5cSJason Baron goto process_status;
10243dedbb5cSJason Baron
1025743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet))
1026743c69e7SNeilBrown status = -ENOBUFS;
1027743c69e7SNeilBrown
1028f279cd00SJason Baron if (sent > 0 || status == 0) {
1029f279cd00SJason Baron req->rq_xmit_bytes_sent += sent;
1030f279cd00SJason Baron if (sent >= req->rq_slen)
1031262965f5SChuck Lever return 0;
1032262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */
1033262965f5SChuck Lever status = -EAGAIN;
10342199700fSTrond Myklebust }
1035262965f5SChuck Lever
10363dedbb5cSJason Baron process_status:
1037262965f5SChuck Lever switch (status) {
1038fba91afbSTrond Myklebust case -ENOTSOCK:
1039fba91afbSTrond Myklebust status = -ENOTCONN;
1040fba91afbSTrond Myklebust /* Should we call xs_close() here? */
1041fba91afbSTrond Myklebust break;
1042b6ddf64fSTrond Myklebust case -EAGAIN:
10437496b59fSTrond Myklebust status = xs_sock_nospace(req);
1044b6ddf64fSTrond Myklebust break;
1045262965f5SChuck Lever case -ENETUNREACH:
10463601c4a9STrond Myklebust case -ENOBUFS:
1047262965f5SChuck Lever case -EPIPE:
1048262965f5SChuck Lever case -ECONNREFUSED:
10493dedbb5cSJason Baron case -EPERM:
1050262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message
1051262965f5SChuck Lever * prompts ECONNREFUSED. */
105213331a55STrond Myklebust break;
105313331a55STrond Myklebust default:
105413331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n",
105513331a55STrond Myklebust -status);
1056262965f5SChuck Lever }
10575fe46e9dSBian Naimeng
1058262965f5SChuck Lever return status;
1059262965f5SChuck Lever }
1060262965f5SChuck Lever
1061e06799f9STrond Myklebust /**
1062262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket
106350f484e2STrond Myklebust * @req: pointer to RPC request
1064262965f5SChuck Lever *
1065262965f5SChuck Lever * Return values:
1066262965f5SChuck Lever * 0: The request has been sent
1067262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to
1068262965f5SChuck Lever * complete the request
1069262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again
107025985edcSLucas De Marchi * other: Some other error occurred, the request was not sent
1071262965f5SChuck Lever *
1072262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up
1073262965f5SChuck Lever * if sendmsg is not able to make progress?
1074262965f5SChuck Lever */
xs_tcp_send_request(struct rpc_rqst * req)1075adfa7144STrond Myklebust static int xs_tcp_send_request(struct rpc_rqst *req)
1076262965f5SChuck Lever {
1077262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt;
1078ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1079262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf;
10807e3d3620STrond Myklebust rpc_fraghdr rm = xs_stream_record_marker(xdr);
10817e3d3620STrond Myklebust unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen;
10829e55eef4SChuck Lever struct msghdr msg = {
10839e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS,
10849e55eef4SChuck Lever };
1085d0afde5fSTrond Myklebust bool vm_wait;
10863f649ab7SKees Cook unsigned int sent;
1087b595bb15SChuck Lever int status;
1088a246b010SChuck Lever
10894cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */
10904cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) {
10914cd34e7cSTrond Myklebust if (transport->sock != NULL)
10924cd34e7cSTrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR);
10934cd34e7cSTrond Myklebust return -ENOTCONN;
10944cd34e7cSTrond Myklebust }
1095d275880aSTrond Myklebust if (!transport->inet)
1096d275880aSTrond Myklebust return -ENOTCONN;
10974cd34e7cSTrond Myklebust
1098262965f5SChuck Lever xs_pktdump("packet data:",
1099262965f5SChuck Lever req->rq_svec->iov_base,
1100262965f5SChuck Lever req->rq_svec->iov_len);
1101a246b010SChuck Lever
11027196dbb0STrond Myklebust if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state))
11037196dbb0STrond Myklebust xs_tcp_set_socket_timeouts(xprt, transport->sock);
11047196dbb0STrond Myklebust
11053b21f757STrond Myklebust xs_set_srcport(transport, transport->sock);
11063b21f757STrond Myklebust
1107a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful
1108a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have
1109262965f5SChuck Lever * called sendmsg(). */
111078215759SChuck Lever req->rq_xtime = ktime_get();
1111d737e5d4STrond Myklebust tcp_sock_set_cork(transport->inet, true);
1112d0afde5fSTrond Myklebust
1113d0afde5fSTrond Myklebust vm_wait = sk_stream_is_writeable(transport->inet) ? true : false;
1114d0afde5fSTrond Myklebust
1115d0afde5fSTrond Myklebust do {
11169e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr,
11177e3d3620STrond Myklebust transport->xmit.offset, rm, &sent);
1118a246b010SChuck Lever
1119262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n",
11206c7a64e5STrond Myklebust xdr->len - transport->xmit.offset, status);
1121262965f5SChuck Lever
1122a246b010SChuck Lever /* If we've sent the entire packet, immediately
1123a246b010SChuck Lever * reset the count of bytes sent. */
11246c7a64e5STrond Myklebust transport->xmit.offset += sent;
11256c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset;
11267e3d3620STrond Myklebust if (likely(req->rq_bytes_sent >= msglen)) {
11276c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset;
11286c7a64e5STrond Myklebust transport->xmit.offset = 0;
1129d737e5d4STrond Myklebust if (atomic_long_read(&xprt->xmit_queuelen) == 1)
1130d737e5d4STrond Myklebust tcp_sock_set_cork(transport->inet, false);
1131a246b010SChuck Lever return 0;
1132a246b010SChuck Lever }
1133262965f5SChuck Lever
11349ffadfbcSTrond Myklebust WARN_ON_ONCE(sent == 0 && status == 0);
11359ffadfbcSTrond Myklebust
1136d0afde5fSTrond Myklebust if (sent > 0)
11379ffadfbcSTrond Myklebust vm_wait = false;
1138d0afde5fSTrond Myklebust
1139d0afde5fSTrond Myklebust } while (status == 0);
1140a246b010SChuck Lever
1141262965f5SChuck Lever switch (status) {
1142fba91afbSTrond Myklebust case -ENOTSOCK:
1143fba91afbSTrond Myklebust status = -ENOTCONN;
1144fba91afbSTrond Myklebust /* Should we call xs_close() here? */
1145fba91afbSTrond Myklebust break;
1146262965f5SChuck Lever case -EAGAIN:
1147d0afde5fSTrond Myklebust status = xs_stream_nospace(req, vm_wait);
1148262965f5SChuck Lever break;
1149262965f5SChuck Lever case -ECONNRESET:
1150262965f5SChuck Lever case -ECONNREFUSED:
1151262965f5SChuck Lever case -ENOTCONN:
11523913c78cSTrond Myklebust case -EADDRINUSE:
1153b5872f0cSTrond Myklebust case -ENOBUFS:
1154b9d2bb2eSTrond Myklebust case -EPIPE:
115513331a55STrond Myklebust break;
115613331a55STrond Myklebust default:
115713331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n",
115813331a55STrond Myklebust -status);
1159a246b010SChuck Lever }
11605fe46e9dSBian Naimeng
1161a246b010SChuck Lever return status;
1162a246b010SChuck Lever }
1163a246b010SChuck Lever
xs_save_old_callbacks(struct sock_xprt * transport,struct sock * sk)11642a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk)
11652a9e1cfaSTrond Myklebust {
11662a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready;
11672a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change;
11682a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space;
11692118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report;
11702a9e1cfaSTrond Myklebust }
11712a9e1cfaSTrond Myklebust
xs_restore_old_callbacks(struct sock_xprt * transport,struct sock * sk)11722a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk)
11732a9e1cfaSTrond Myklebust {
11742a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready;
11752a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change;
11762a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space;
11772118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report;
11782118071dSTrond Myklebust }
11792118071dSTrond Myklebust
xs_sock_reset_state_flags(struct rpc_xprt * xprt)118042d42a5bSTrond Myklebust static void xs_sock_reset_state_flags(struct rpc_xprt *xprt)
118142d42a5bSTrond Myklebust {
118242d42a5bSTrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
118342d42a5bSTrond Myklebust
118442d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state);
11854f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state);
11864f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state);
11874f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state);
11882790a624STrond Myklebust clear_bit(XPRT_SOCK_NOSPACE, &transport->sock_state);
11894f8943f8STrond Myklebust }
11904f8943f8STrond Myklebust
xs_run_error_worker(struct sock_xprt * transport,unsigned int nr)11914f8943f8STrond Myklebust static void xs_run_error_worker(struct sock_xprt *transport, unsigned int nr)
11924f8943f8STrond Myklebust {
11934f8943f8STrond Myklebust set_bit(nr, &transport->sock_state);
11944f8943f8STrond Myklebust queue_work(xprtiod_workqueue, &transport->error_worker);
119542d42a5bSTrond Myklebust }
119642d42a5bSTrond Myklebust
xs_sock_reset_connection_flags(struct rpc_xprt * xprt)1197b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt)
1198b70ae915STrond Myklebust {
1199d896ba83STrond Myklebust xprt->connect_cookie++;
1200b70ae915STrond Myklebust smp_mb__before_atomic();
1201b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
1202b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state);
120342d42a5bSTrond Myklebust xs_sock_reset_state_flags(xprt);
1204b70ae915STrond Myklebust smp_mb__after_atomic();
1205b70ae915STrond Myklebust }
1206b70ae915STrond Myklebust
12072118071dSTrond Myklebust /**
12082118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors
12092118071dSTrond Myklebust * @sk: socket
12102118071dSTrond Myklebust *
12112118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task
12122118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err.
12132118071dSTrond Myklebust */
xs_error_report(struct sock * sk)12142118071dSTrond Myklebust static void xs_error_report(struct sock *sk)
12152118071dSTrond Myklebust {
12164f8943f8STrond Myklebust struct sock_xprt *transport;
12172118071dSTrond Myklebust struct rpc_xprt *xprt;
12182118071dSTrond Myklebust
12192118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk)))
1220ea9afca8STrond Myklebust return;
12212118071dSTrond Myklebust
12224f8943f8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt);
1223af84537dSBenjamin Coddington transport->xprt_err = -sk->sk_err;
1224af84537dSBenjamin Coddington if (transport->xprt_err == 0)
1225ea9afca8STrond Myklebust return;
12262118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n",
1227af84537dSBenjamin Coddington xprt, -transport->xprt_err);
1228af84537dSBenjamin Coddington trace_rpc_socket_error(xprt, sk->sk_socket, transport->xprt_err);
1229af84537dSBenjamin Coddington
1230af84537dSBenjamin Coddington /* barrier ensures xprt_err is set before XPRT_SOCK_WAKE_ERROR */
1231af84537dSBenjamin Coddington smp_mb__before_atomic();
12324f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_ERROR);
12332a9e1cfaSTrond Myklebust }
12342a9e1cfaSTrond Myklebust
xs_reset_transport(struct sock_xprt * transport)1235fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport)
1236a246b010SChuck Lever {
1237ee0ac0c2SChuck Lever struct socket *sock = transport->sock;
1238ee0ac0c2SChuck Lever struct sock *sk = transport->inet;
12396cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt;
1240a73881c9STrond Myklebust struct file *filp = transport->file;
1241a246b010SChuck Lever
1242fe315e76SChuck Lever if (sk == NULL)
1243fe315e76SChuck Lever return;
1244f0043206STrond Myklebust /*
1245f0043206STrond Myklebust * Make sure we're calling this in a context from which it is safe
1246f0043206STrond Myklebust * to call __fput_sync(). In practice that means rpciod and the
1247f0043206STrond Myklebust * system workqueue.
1248f0043206STrond Myklebust */
1249f0043206STrond Myklebust if (!(current->flags & PF_WQ_WORKER)) {
1250f0043206STrond Myklebust WARN_ON_ONCE(1);
1251f0043206STrond Myklebust set_bit(XPRT_CLOSE_WAIT, &xprt->state);
1252f0043206STrond Myklebust return;
1253f0043206STrond Myklebust }
12549903cd1cSChuck Lever
1255264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper))
1256264d1df3SJeff Layton sk_clear_memalloc(sk);
1257264d1df3SJeff Layton
125875eb6af7SChuck Lever tls_handshake_cancel(sk);
125975eb6af7SChuck Lever
126009939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR);
126109939204STrond Myklebust
1262edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex);
1263ea9afca8STrond Myklebust lock_sock(sk);
1264ee0ac0c2SChuck Lever transport->inet = NULL;
1265ee0ac0c2SChuck Lever transport->sock = NULL;
1266a73881c9STrond Myklebust transport->file = NULL;
1267a246b010SChuck Lever
1268a246b010SChuck Lever sk->sk_user_data = NULL;
12692a9e1cfaSTrond Myklebust
12702a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk);
12710c78789eSTrond Myklebust xprt_clear_connected(xprt);
12726cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt);
1273ae053551STrond Myklebust /* Reset stream record info */
1274ae053551STrond Myklebust xs_stream_reset_connect(transport);
1275ea9afca8STrond Myklebust release_sock(sk);
1276edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex);
1277a246b010SChuck Lever
12786cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock);
1279f0043206STrond Myklebust __fput_sync(filp);
12800445f92cSTrond Myklebust
12810445f92cSTrond Myklebust xprt_disconnect_done(xprt);
1282fe315e76SChuck Lever }
1283fe315e76SChuck Lever
1284fe315e76SChuck Lever /**
1285fe315e76SChuck Lever * xs_close - close a socket
1286fe315e76SChuck Lever * @xprt: transport
1287fe315e76SChuck Lever *
1288fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains
1289fe315e76SChuck Lever * on the server we want to save.
1290f75e6745STrond Myklebust *
1291f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with
1292f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer.
1293fe315e76SChuck Lever */
xs_close(struct rpc_xprt * xprt)1294fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt)
1295fe315e76SChuck Lever {
1296fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1297fe315e76SChuck Lever
1298fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt);
1299fe315e76SChuck Lever
13005dd5ad68SChuck Lever if (transport->sock)
13015dd5ad68SChuck Lever tls_handshake_close(transport->sock);
1302fe315e76SChuck Lever xs_reset_transport(transport);
130361d0a8e6SNeil Brown xprt->reestablish_timeout = 0;
1304a246b010SChuck Lever }
1305a246b010SChuck Lever
xs_inject_disconnect(struct rpc_xprt * xprt)13064a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt)
13074a068258SChuck Lever {
13084a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n",
13094a068258SChuck Lever xprt);
13104a068258SChuck Lever xprt_disconnect_done(xprt);
13114a068258SChuck Lever }
13124a068258SChuck Lever
xs_xprt_free(struct rpc_xprt * xprt)1313315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt)
1314315f3812SKinglong Mee {
1315315f3812SKinglong Mee xs_free_peer_addresses(xprt);
1316315f3812SKinglong Mee xprt_free(xprt);
1317315f3812SKinglong Mee }
1318315f3812SKinglong Mee
13199903cd1cSChuck Lever /**
13209903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport
13219903cd1cSChuck Lever * @xprt: doomed transport
13229903cd1cSChuck Lever *
13239903cd1cSChuck Lever */
xs_destroy(struct rpc_xprt * xprt)13249903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt)
1325a246b010SChuck Lever {
132603c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt,
132703c78827STrond Myklebust struct sock_xprt, xprt);
13289903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt);
13299903cd1cSChuck Lever
133003c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker);
1331a1311d87STrond Myklebust xs_close(xprt);
1332edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker);
1333b5e92419STrond Myklebust cancel_work_sync(&transport->error_worker);
1334315f3812SKinglong Mee xs_xprt_free(xprt);
1335a1311d87STrond Myklebust module_put(THIS_MODULE);
1336a246b010SChuck Lever }
1337a246b010SChuck Lever
13389903cd1cSChuck Lever /**
1339f9b2ee71STrond Myklebust * xs_udp_data_read_skb - receive callback for UDP sockets
1340f9b2ee71STrond Myklebust * @xprt: transport
1341f9b2ee71STrond Myklebust * @sk: socket
1342f9b2ee71STrond Myklebust * @skb: skbuff
13439903cd1cSChuck Lever *
1344a246b010SChuck Lever */
xs_udp_data_read_skb(struct rpc_xprt * xprt,struct sock * sk,struct sk_buff * skb)1345f9b2ee71STrond Myklebust static void xs_udp_data_read_skb(struct rpc_xprt *xprt,
1346f9b2ee71STrond Myklebust struct sock *sk,
1347f9b2ee71STrond Myklebust struct sk_buff *skb)
1348a246b010SChuck Lever {
1349a246b010SChuck Lever struct rpc_task *task;
1350a246b010SChuck Lever struct rpc_rqst *rovr;
1351f9b2ee71STrond Myklebust int repsize, copied;
1352d8ed029dSAlexey Dobriyan u32 _xid;
1353d8ed029dSAlexey Dobriyan __be32 *xp;
1354a246b010SChuck Lever
13551da8c681SWillem de Bruijn repsize = skb->len;
1356a246b010SChuck Lever if (repsize < 4) {
13579903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize);
1358f9b2ee71STrond Myklebust return;
1359a246b010SChuck Lever }
1360a246b010SChuck Lever
1361a246b010SChuck Lever /* Copy the XID from the skb... */
13621da8c681SWillem de Bruijn xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid);
1363a246b010SChuck Lever if (xp == NULL)
1364f9b2ee71STrond Myklebust return;
1365a246b010SChuck Lever
1366a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */
136775c84151STrond Myklebust spin_lock(&xprt->queue_lock);
1368a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp);
1369a246b010SChuck Lever if (!rovr)
1370a246b010SChuck Lever goto out_unlock;
1371729749bbSTrond Myklebust xprt_pin_rqst(rovr);
1372ecd465eeSChuck Lever xprt_update_rtt(rovr->rq_task);
137375c84151STrond Myklebust spin_unlock(&xprt->queue_lock);
1374a246b010SChuck Lever task = rovr->rq_task;
1375a246b010SChuck Lever
1376a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize)
1377a246b010SChuck Lever copied = repsize;
1378a246b010SChuck Lever
1379a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */
13801781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) {
138175c84151STrond Myklebust spin_lock(&xprt->queue_lock);
13820afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INERRORS);
1383729749bbSTrond Myklebust goto out_unpin;
13841781f7f5SHerbert Xu }
13851781f7f5SHerbert Xu
1386a246b010SChuck Lever
1387b5e92419STrond Myklebust spin_lock(&xprt->transport_lock);
13886a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied);
1389b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock);
139075c84151STrond Myklebust spin_lock(&xprt->queue_lock);
13911570c1e4SChuck Lever xprt_complete_rqst(task, copied);
13920afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS);
1393729749bbSTrond Myklebust out_unpin:
1394729749bbSTrond Myklebust xprt_unpin_rqst(rovr);
1395a246b010SChuck Lever out_unlock:
139675c84151STrond Myklebust spin_unlock(&xprt->queue_lock);
1397f9b2ee71STrond Myklebust }
1398f9b2ee71STrond Myklebust
xs_udp_data_receive(struct sock_xprt * transport)1399f9b2ee71STrond Myklebust static void xs_udp_data_receive(struct sock_xprt *transport)
1400f9b2ee71STrond Myklebust {
1401f9b2ee71STrond Myklebust struct sk_buff *skb;
1402f9b2ee71STrond Myklebust struct sock *sk;
1403f9b2ee71STrond Myklebust int err;
1404f9b2ee71STrond Myklebust
1405f9b2ee71STrond Myklebust mutex_lock(&transport->recv_mutex);
1406f9b2ee71STrond Myklebust sk = transport->inet;
1407f9b2ee71STrond Myklebust if (sk == NULL)
1408f9b2ee71STrond Myklebust goto out;
1409f9b2ee71STrond Myklebust for (;;) {
1410ec095263SOliver Hartkopp skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
14114f546149STrond Myklebust if (skb == NULL)
14124f546149STrond Myklebust break;
1413f9b2ee71STrond Myklebust xs_udp_data_read_skb(&transport->xprt, sk, skb);
1414850cbaddSPaolo Abeni consume_skb(skb);
14150af3442aSTrond Myklebust cond_resched();
1416f9b2ee71STrond Myklebust }
14170ffe86f4STrond Myklebust xs_poll_check_readable(transport);
1418a246b010SChuck Lever out:
1419f9b2ee71STrond Myklebust mutex_unlock(&transport->recv_mutex);
1420f9b2ee71STrond Myklebust }
1421f9b2ee71STrond Myklebust
xs_udp_data_receive_workfn(struct work_struct * work)1422f9b2ee71STrond Myklebust static void xs_udp_data_receive_workfn(struct work_struct *work)
1423f9b2ee71STrond Myklebust {
1424f9b2ee71STrond Myklebust struct sock_xprt *transport =
1425f9b2ee71STrond Myklebust container_of(work, struct sock_xprt, recv_worker);
1426a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save();
1427a1231fdaSTrond Myklebust
1428f9b2ee71STrond Myklebust xs_udp_data_receive(transport);
1429a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags);
1430f9b2ee71STrond Myklebust }
1431f9b2ee71STrond Myklebust
1432f9b2ee71STrond Myklebust /**
1433f67939e4SChuck Lever * xs_data_ready - "data ready" callback for sockets
1434f9b2ee71STrond Myklebust * @sk: socket with data to read
1435f9b2ee71STrond Myklebust *
1436f9b2ee71STrond Myklebust */
xs_data_ready(struct sock * sk)1437f9b2ee71STrond Myklebust static void xs_data_ready(struct sock *sk)
1438f9b2ee71STrond Myklebust {
1439f9b2ee71STrond Myklebust struct rpc_xprt *xprt;
1440f9b2ee71STrond Myklebust
144140e0b090SPeilin Ye trace_sk_data_ready(sk);
144240e0b090SPeilin Ye
1443f9b2ee71STrond Myklebust xprt = xprt_from_sock(sk);
1444f9b2ee71STrond Myklebust if (xprt != NULL) {
1445f9b2ee71STrond Myklebust struct sock_xprt *transport = container_of(xprt,
1446f9b2ee71STrond Myklebust struct sock_xprt, xprt);
1447f67939e4SChuck Lever
1448f67939e4SChuck Lever trace_xs_data_ready(xprt);
1449f67939e4SChuck Lever
14505157b956STrond Myklebust transport->old_data_ready(sk);
14510d3ca07fSChuck Lever
14520d3ca07fSChuck Lever if (test_bit(XPRT_SOCK_IGNORE_RECV, &transport->sock_state))
14530d3ca07fSChuck Lever return;
14540d3ca07fSChuck Lever
14555157b956STrond Myklebust /* Any data means we had a useful conversation, so
14565157b956STrond Myklebust * then we don't need to delay the next reconnect
14575157b956STrond Myklebust */
14585157b956STrond Myklebust if (xprt->reestablish_timeout)
14595157b956STrond Myklebust xprt->reestablish_timeout = 0;
146042d42a5bSTrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state))
146140a5f1b1STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker);
1462f9b2ee71STrond Myklebust }
1463a246b010SChuck Lever }
1464a246b010SChuck Lever
1465a519fc7aSTrond Myklebust /*
1466a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending
1467a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT
1468a519fc7aSTrond Myklebust */
xs_tcp_force_close(struct rpc_xprt * xprt)1469a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt)
1470a519fc7aSTrond Myklebust {
1471a519fc7aSTrond Myklebust xprt_force_disconnect(xprt);
1472a519fc7aSTrond Myklebust }
1473a519fc7aSTrond Myklebust
14749e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL)
xs_tcp_bc_maxpayload(struct rpc_xprt * xprt)14756b26cc8cSChuck Lever static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt)
14766b26cc8cSChuck Lever {
14776b26cc8cSChuck Lever return PAGE_SIZE;
14786b26cc8cSChuck Lever }
14799e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */
148044b98efdSRicardo Labiaga
14819903cd1cSChuck Lever /**
1482efce2d0bSTrond Myklebust * xs_local_state_change - callback to handle AF_LOCAL socket state changes
1483efce2d0bSTrond Myklebust * @sk: socket whose state has changed
1484efce2d0bSTrond Myklebust *
1485efce2d0bSTrond Myklebust */
xs_local_state_change(struct sock * sk)1486efce2d0bSTrond Myklebust static void xs_local_state_change(struct sock *sk)
1487efce2d0bSTrond Myklebust {
1488efce2d0bSTrond Myklebust struct rpc_xprt *xprt;
1489efce2d0bSTrond Myklebust struct sock_xprt *transport;
1490efce2d0bSTrond Myklebust
1491efce2d0bSTrond Myklebust if (!(xprt = xprt_from_sock(sk)))
1492efce2d0bSTrond Myklebust return;
1493efce2d0bSTrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt);
1494efce2d0bSTrond Myklebust if (sk->sk_shutdown & SHUTDOWN_MASK) {
1495efce2d0bSTrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state);
1496efce2d0bSTrond Myklebust /* Trigger the socket release */
1497efce2d0bSTrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
1498efce2d0bSTrond Myklebust }
1499efce2d0bSTrond Myklebust }
1500efce2d0bSTrond Myklebust
1501efce2d0bSTrond Myklebust /**
15029903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes
15039903cd1cSChuck Lever * @sk: socket whose state has changed
15049903cd1cSChuck Lever *
15059903cd1cSChuck Lever */
xs_tcp_state_change(struct sock * sk)15069903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk)
1507a246b010SChuck Lever {
1508a246b010SChuck Lever struct rpc_xprt *xprt;
15090fdea1e8STrond Myklebust struct sock_xprt *transport;
1510a246b010SChuck Lever
1511a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk)))
1512ea9afca8STrond Myklebust return;
15139903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt);
1514669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n",
1515a246b010SChuck Lever sk->sk_state, xprt_connected(xprt),
1516a246b010SChuck Lever sock_flag(sk, SOCK_DEAD),
1517669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED),
1518669502ffSAndy Chittenden sk->sk_shutdown);
1519a246b010SChuck Lever
15200fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt);
152140b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket);
1522a246b010SChuck Lever switch (sk->sk_state) {
1523a246b010SChuck Lever case TCP_ESTABLISHED:
1524a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) {
15258b71798cSTrond Myklebust xprt->connect_cookie++;
15260fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
15270fdea1e8STrond Myklebust xprt_clear_connecting(xprt);
152851971139SChuck Lever
15293968a8a5SChuck Lever xprt->stat.connect_count++;
15303968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies -
15313968a8a5SChuck Lever xprt->stat.connect_start;
15324f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_PENDING);
1533a246b010SChuck Lever }
1534a246b010SChuck Lever break;
15353b948ae5STrond Myklebust case TCP_FIN_WAIT1:
15363b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */
15377c1d71cfSTrond Myklebust xprt->connect_cookie++;
1538663b8858STrond Myklebust xprt->reestablish_timeout = 0;
15393b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state);
15404e857c58SPeter Zijlstra smp_mb__before_atomic();
15413b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state);
1542ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
15434e857c58SPeter Zijlstra smp_mb__after_atomic();
1544a246b010SChuck Lever break;
1545632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT:
15463b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */
15477c1d71cfSTrond Myklebust xprt->connect_cookie++;
1548d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state);
15494f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
1550df561f66SGustavo A. R. Silva fallthrough;
1551663b8858STrond Myklebust case TCP_CLOSING:
1552663b8858STrond Myklebust /*
1553663b8858STrond Myklebust * If the server closed down the connection, make sure that
1554663b8858STrond Myklebust * we back off before reconnecting
1555663b8858STrond Myklebust */
1556663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
1557663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
15583b948ae5STrond Myklebust break;
15593b948ae5STrond Myklebust case TCP_LAST_ACK:
1560670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state);
15614e857c58SPeter Zijlstra smp_mb__before_atomic();
15623b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state);
15634e857c58SPeter Zijlstra smp_mb__after_atomic();
15643b948ae5STrond Myklebust break;
15653b948ae5STrond Myklebust case TCP_CLOSE:
15660fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING,
15670fdea1e8STrond Myklebust &transport->sock_state))
15680fdea1e8STrond Myklebust xprt_clear_connecting(xprt);
15699b30889cSTrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state);
15709b30889cSTrond Myklebust /* Trigger the socket release */
15714f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT);
1572a246b010SChuck Lever }
1573a246b010SChuck Lever }
1574a246b010SChuck Lever
xs_write_space(struct sock * sk)15751f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk)
15761f0fa154SIlpo Järvinen {
15774f8943f8STrond Myklebust struct sock_xprt *transport;
15781f0fa154SIlpo Järvinen struct rpc_xprt *xprt;
15791f0fa154SIlpo Järvinen
158013331a55STrond Myklebust if (!sk->sk_socket)
15811f0fa154SIlpo Järvinen return;
158213331a55STrond Myklebust clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
15831f0fa154SIlpo Järvinen
15841f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk))))
15851f0fa154SIlpo Järvinen return;
15864f8943f8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt);
15872790a624STrond Myklebust if (!test_and_clear_bit(XPRT_SOCK_NOSPACE, &transport->sock_state))
15882790a624STrond Myklebust return;
15894f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_WRITE);
1590c544577dSTrond Myklebust sk->sk_write_pending--;
15911f0fa154SIlpo Järvinen }
15921f0fa154SIlpo Järvinen
15932a9e1cfaSTrond Myklebust /**
1594c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space
1595c7b2cae8SChuck Lever * becomes available
15969903cd1cSChuck Lever * @sk: socket whose state has changed
15979903cd1cSChuck Lever *
1598a246b010SChuck Lever * Called when more output buffer space is available for this socket.
1599a246b010SChuck Lever * We try not to wake our writers until they can make "significant"
1600c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1601a246b010SChuck Lever * with a bunch of small requests.
1602a246b010SChuck Lever */
xs_udp_write_space(struct sock * sk)1603c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk)
1604a246b010SChuck Lever {
1605c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */
16061f0fa154SIlpo Järvinen if (sock_writeable(sk))
16071f0fa154SIlpo Järvinen xs_write_space(sk);
1608c7b2cae8SChuck Lever }
1609c7b2cae8SChuck Lever
1610c7b2cae8SChuck Lever /**
1611c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space
1612c7b2cae8SChuck Lever * becomes available
1613c7b2cae8SChuck Lever * @sk: socket whose state has changed
1614c7b2cae8SChuck Lever *
1615c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket.
1616c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant"
1617c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg
1618c7b2cae8SChuck Lever * with a bunch of small requests.
1619c7b2cae8SChuck Lever */
xs_tcp_write_space(struct sock * sk)1620c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk)
1621c7b2cae8SChuck Lever {
1622c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */
162364dc6130SEric Dumazet if (sk_stream_is_writeable(sk))
16241f0fa154SIlpo Järvinen xs_write_space(sk);
1625a246b010SChuck Lever }
1626a246b010SChuck Lever
xs_udp_do_set_buffer_size(struct rpc_xprt * xprt)1627470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt)
1628a246b010SChuck Lever {
1629ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
1630ee0ac0c2SChuck Lever struct sock *sk = transport->inet;
1631a246b010SChuck Lever
16327c6e066eSChuck Lever if (transport->rcvsize) {
1633a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
16347c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2;
1635a246b010SChuck Lever }
16367c6e066eSChuck Lever if (transport->sndsize) {
1637a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
16387c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2;
1639a246b010SChuck Lever sk->sk_write_space(sk);
1640a246b010SChuck Lever }
1641a246b010SChuck Lever }
1642a246b010SChuck Lever
164343118c29SChuck Lever /**
1644470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits
164543118c29SChuck Lever * @xprt: generic transport
1646470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes
1647470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes
164843118c29SChuck Lever *
1649470056c2SChuck Lever * Set socket send and receive buffer size limits.
165043118c29SChuck Lever */
xs_udp_set_buffer_size(struct rpc_xprt * xprt,size_t sndsize,size_t rcvsize)1651470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize)
165243118c29SChuck Lever {
16537c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
16547c6e066eSChuck Lever
16557c6e066eSChuck Lever transport->sndsize = 0;
1656470056c2SChuck Lever if (sndsize)
16577c6e066eSChuck Lever transport->sndsize = sndsize + 1024;
16587c6e066eSChuck Lever transport->rcvsize = 0;
1659470056c2SChuck Lever if (rcvsize)
16607c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024;
1661470056c2SChuck Lever
1662470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt);
166343118c29SChuck Lever }
166443118c29SChuck Lever
166546c0ee8bSChuck Lever /**
166646c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport
1667acf0a39fSChuck Lever * @xprt: controlling transport
166846c0ee8bSChuck Lever * @task: task that timed out
166946c0ee8bSChuck Lever *
167046c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred.
167146c0ee8bSChuck Lever */
xs_udp_timer(struct rpc_xprt * xprt,struct rpc_task * task)16726a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task)
167346c0ee8bSChuck Lever {
1674b5e92419STrond Myklebust spin_lock(&xprt->transport_lock);
16756a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT);
1676b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock);
167746c0ee8bSChuck Lever }
167846c0ee8bSChuck Lever
xs_get_random_port(void)1679826799e6SJ. Bruce Fields static int xs_get_random_port(void)
1680b85d8806SChuck Lever {
1681826799e6SJ. Bruce Fields unsigned short min = xprt_min_resvport, max = xprt_max_resvport;
1682826799e6SJ. Bruce Fields unsigned short range;
1683826799e6SJ. Bruce Fields unsigned short rand;
1684826799e6SJ. Bruce Fields
1685826799e6SJ. Bruce Fields if (max < min)
1686826799e6SJ. Bruce Fields return -EADDRINUSE;
1687826799e6SJ. Bruce Fields range = max - min + 1;
16888032bf12SJason A. Donenfeld rand = get_random_u32_below(range);
1689826799e6SJ. Bruce Fields return rand + min;
1690b85d8806SChuck Lever }
1691b85d8806SChuck Lever
xs_sock_getport(struct socket * sock)16924dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock)
16934dda9c8aSTrond Myklebust {
16944dda9c8aSTrond Myklebust struct sockaddr_storage buf;
16954dda9c8aSTrond Myklebust unsigned short port = 0;
16964dda9c8aSTrond Myklebust
16979b2c45d4SDenys Vlasenko if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0)
16984dda9c8aSTrond Myklebust goto out;
16994dda9c8aSTrond Myklebust switch (buf.ss_family) {
17004dda9c8aSTrond Myklebust case AF_INET6:
17014dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port);
17024dda9c8aSTrond Myklebust break;
17034dda9c8aSTrond Myklebust case AF_INET:
17044dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port);
17054dda9c8aSTrond Myklebust }
17064dda9c8aSTrond Myklebust out:
17074dda9c8aSTrond Myklebust return port;
17084dda9c8aSTrond Myklebust }
17094dda9c8aSTrond Myklebust
17104dda9c8aSTrond Myklebust /**
171192200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address
171292200412SChuck Lever * @xprt: generic transport
171392200412SChuck Lever * @port: new port number
171492200412SChuck Lever *
171592200412SChuck Lever */
xs_set_port(struct rpc_xprt * xprt,unsigned short port)171692200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port)
171792200412SChuck Lever {
171892200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port);
1719c4efcb1dSChuck Lever
17209dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port);
17219dc3b095SChuck Lever xs_update_peer_port(xprt);
172292200412SChuck Lever }
172392200412SChuck Lever
xs_set_srcport(struct sock_xprt * transport,struct socket * sock)17244dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock)
17254dda9c8aSTrond Myklebust {
1726e6237b6fSTrond Myklebust if (transport->srcport == 0 && transport->xprt.reuseport)
17274dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock);
17284dda9c8aSTrond Myklebust }
17294dda9c8aSTrond Myklebust
xs_get_srcport(struct sock_xprt * transport)1730826799e6SJ. Bruce Fields static int xs_get_srcport(struct sock_xprt *transport)
173167a391d7STrond Myklebust {
1732826799e6SJ. Bruce Fields int port = transport->srcport;
173367a391d7STrond Myklebust
173467a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport)
173567a391d7STrond Myklebust port = xs_get_random_port();
173667a391d7STrond Myklebust return port;
173767a391d7STrond Myklebust }
173867a391d7STrond Myklebust
xs_sock_srcport(struct rpc_xprt * xprt)1739421ab1beSTrond Myklebust static unsigned short xs_sock_srcport(struct rpc_xprt *xprt)
1740a8482488SOlga Kornievskaia {
1741a8482488SOlga Kornievskaia struct sock_xprt *sock = container_of(xprt, struct sock_xprt, xprt);
1742b49ea673SNeilBrown unsigned short ret = 0;
1743b49ea673SNeilBrown mutex_lock(&sock->recv_mutex);
1744b49ea673SNeilBrown if (sock->sock)
1745b49ea673SNeilBrown ret = xs_sock_getport(sock->sock);
1746b49ea673SNeilBrown mutex_unlock(&sock->recv_mutex);
1747b49ea673SNeilBrown return ret;
1748a8482488SOlga Kornievskaia }
1749421ab1beSTrond Myklebust
xs_sock_srcaddr(struct rpc_xprt * xprt,char * buf,size_t buflen)1750421ab1beSTrond Myklebust static int xs_sock_srcaddr(struct rpc_xprt *xprt, char *buf, size_t buflen)
1751421ab1beSTrond Myklebust {
1752421ab1beSTrond Myklebust struct sock_xprt *sock = container_of(xprt, struct sock_xprt, xprt);
1753421ab1beSTrond Myklebust union {
1754421ab1beSTrond Myklebust struct sockaddr sa;
1755421ab1beSTrond Myklebust struct sockaddr_storage st;
1756421ab1beSTrond Myklebust } saddr;
1757421ab1beSTrond Myklebust int ret = -ENOTCONN;
1758421ab1beSTrond Myklebust
1759421ab1beSTrond Myklebust mutex_lock(&sock->recv_mutex);
1760421ab1beSTrond Myklebust if (sock->sock) {
1761421ab1beSTrond Myklebust ret = kernel_getsockname(sock->sock, &saddr.sa);
1762421ab1beSTrond Myklebust if (ret >= 0)
1763421ab1beSTrond Myklebust ret = snprintf(buf, buflen, "%pISc", &saddr.sa);
1764421ab1beSTrond Myklebust }
1765421ab1beSTrond Myklebust mutex_unlock(&sock->recv_mutex);
1766421ab1beSTrond Myklebust return ret;
1767421ab1beSTrond Myklebust }
1768a8482488SOlga Kornievskaia
xs_next_srcport(struct sock_xprt * transport,unsigned short port)1769baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port)
177067a391d7STrond Myklebust {
1771fbfffbd5SChuck Lever if (transport->srcport != 0)
1772fbfffbd5SChuck Lever transport->srcport = 0;
177367a391d7STrond Myklebust if (!transport->xprt.resvport)
177467a391d7STrond Myklebust return 0;
177567a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport)
177667a391d7STrond Myklebust return xprt_max_resvport;
177767a391d7STrond Myklebust return --port;
177867a391d7STrond Myklebust }
xs_bind(struct sock_xprt * transport,struct socket * sock)1779beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock)
1780a246b010SChuck Lever {
1781beb59b68SPavel Emelyanov struct sockaddr_storage myaddr;
178267a391d7STrond Myklebust int err, nloop = 0;
1783826799e6SJ. Bruce Fields int port = xs_get_srcport(transport);
178467a391d7STrond Myklebust unsigned short last;
1785a246b010SChuck Lever
17860f7a622cSChris Perl /*
17870f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 &&
17880f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local
17890f7a622cSChris Perl * port selection happen implicitly when the socket is used
17900f7a622cSChris Perl * (for example at connect time).
17910f7a622cSChris Perl *
17920f7a622cSChris Perl * This ensures that we can continue to establish TCP
17930f7a622cSChris Perl * connections even when all local ephemeral ports are already
17940f7a622cSChris Perl * a part of some TCP connection. This makes no difference
179512b20ce3SBhaskar Chowdhury * for UDP sockets, but also doesn't harm them.
17960f7a622cSChris Perl *
17970f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 &&
17980f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will
17990f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed.
18000f7a622cSChris Perl */
1801826799e6SJ. Bruce Fields if (port <= 0)
1802826799e6SJ. Bruce Fields return port;
18030f7a622cSChris Perl
1804beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen);
1805a246b010SChuck Lever do {
1806beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port);
1807e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr,
1808beb59b68SPavel Emelyanov transport->xprt.addrlen);
1809a246b010SChuck Lever if (err == 0) {
1810bc1c56e9SNeilBrown if (transport->xprt.reuseport)
1811fbfffbd5SChuck Lever transport->srcport = port;
1812d3bc9a1dSFrank van Maarseveen break;
1813a246b010SChuck Lever }
181467a391d7STrond Myklebust last = port;
1815baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port);
181667a391d7STrond Myklebust if (port > last)
181767a391d7STrond Myklebust nloop++;
181867a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2);
1819beb59b68SPavel Emelyanov
18204232e863SChuck Lever if (myaddr.ss_family == AF_INET)
1821beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__,
1822beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr,
1823beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err);
1824beb59b68SPavel Emelyanov else
1825beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__,
1826beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr,
18277dc753f0SChuck Lever port, err ? "failed" : "ok", err);
1828a246b010SChuck Lever return err;
1829a246b010SChuck Lever }
1830a246b010SChuck Lever
1831176e21eeSChuck Lever /*
1832176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets
1833176e21eeSChuck Lever */
xs_local_rpcbind(struct rpc_task * task)1834176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task)
1835176e21eeSChuck Lever {
1836fb43d172STrond Myklebust xprt_set_bound(task->tk_xprt);
1837176e21eeSChuck Lever }
1838176e21eeSChuck Lever
xs_local_set_port(struct rpc_xprt * xprt,unsigned short port)1839176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port)
1840176e21eeSChuck Lever {
1841176e21eeSChuck Lever }
1842a246b010SChuck Lever
1843ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC
1844064a9177SNeilBrown static struct lock_class_key xs_key[3];
1845064a9177SNeilBrown static struct lock_class_key xs_slock_key[3];
1846ed07536eSPeter Zijlstra
xs_reclassify_socketu(struct socket * sock)1847176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock)
1848176e21eeSChuck Lever {
1849176e21eeSChuck Lever struct sock *sk = sock->sk;
1850176e21eeSChuck Lever
1851176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC",
1852064a9177SNeilBrown &xs_slock_key[0], "sk_lock-AF_LOCAL-RPC", &xs_key[0]);
1853176e21eeSChuck Lever }
1854176e21eeSChuck Lever
xs_reclassify_socket4(struct socket * sock)18558945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock)
1856ed07536eSPeter Zijlstra {
1857ed07536eSPeter Zijlstra struct sock *sk = sock->sk;
18588945ee5eSChuck Lever
18598945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC",
1860064a9177SNeilBrown &xs_slock_key[1], "sk_lock-AF_INET-RPC", &xs_key[1]);
1861ed07536eSPeter Zijlstra }
18628945ee5eSChuck Lever
xs_reclassify_socket6(struct socket * sock)18638945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock)
18648945ee5eSChuck Lever {
18658945ee5eSChuck Lever struct sock *sk = sock->sk;
18668945ee5eSChuck Lever
18678945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC",
1868064a9177SNeilBrown &xs_slock_key[2], "sk_lock-AF_INET6-RPC", &xs_key[2]);
1869ed07536eSPeter Zijlstra }
18706bc9638aSPavel Emelyanov
xs_reclassify_socket(int family,struct socket * sock)18716bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock)
18726bc9638aSPavel Emelyanov {
1873fafc4e1eSHannes Frederic Sowa if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk)))
18741b7a1819SWeston Andros Adamson return;
18751b7a1819SWeston Andros Adamson
18764232e863SChuck Lever switch (family) {
1877176e21eeSChuck Lever case AF_LOCAL:
1878176e21eeSChuck Lever xs_reclassify_socketu(sock);
1879176e21eeSChuck Lever break;
18804232e863SChuck Lever case AF_INET:
18816bc9638aSPavel Emelyanov xs_reclassify_socket4(sock);
18824232e863SChuck Lever break;
18834232e863SChuck Lever case AF_INET6:
18846bc9638aSPavel Emelyanov xs_reclassify_socket6(sock);
18854232e863SChuck Lever break;
18864232e863SChuck Lever }
18876bc9638aSPavel Emelyanov }
1888ed07536eSPeter Zijlstra #else
xs_reclassify_socket(int family,struct socket * sock)18896bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock)
18906bc9638aSPavel Emelyanov {
18916bc9638aSPavel Emelyanov }
1892ed07536eSPeter Zijlstra #endif
1893ed07536eSPeter Zijlstra
xs_dummy_setup_socket(struct work_struct * work)189493dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work)
189593dc41bdSNeilBrown {
189693dc41bdSNeilBrown }
189793dc41bdSNeilBrown
xs_create_sock(struct rpc_xprt * xprt,struct sock_xprt * transport,int family,int type,int protocol,bool reuseport)18986bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt,
18994dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type,
19004dda9c8aSTrond Myklebust int protocol, bool reuseport)
190122f79326SPavel Emelyanov {
1902a73881c9STrond Myklebust struct file *filp;
190322f79326SPavel Emelyanov struct socket *sock;
190422f79326SPavel Emelyanov int err;
190522f79326SPavel Emelyanov
19066bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1);
190722f79326SPavel Emelyanov if (err < 0) {
190822f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n",
190922f79326SPavel Emelyanov protocol, -err);
191022f79326SPavel Emelyanov goto out;
191122f79326SPavel Emelyanov }
19126bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock);
191322f79326SPavel Emelyanov
19144dda9c8aSTrond Myklebust if (reuseport)
1915fe31a326SChristoph Hellwig sock_set_reuseport(sock->sk);
19164dda9c8aSTrond Myklebust
19174cea288aSBen Hutchings err = xs_bind(transport, sock);
19184cea288aSBen Hutchings if (err) {
191922f79326SPavel Emelyanov sock_release(sock);
192022f79326SPavel Emelyanov goto out;
192122f79326SPavel Emelyanov }
192222f79326SPavel Emelyanov
1923a73881c9STrond Myklebust filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1924a73881c9STrond Myklebust if (IS_ERR(filp))
1925a73881c9STrond Myklebust return ERR_CAST(filp);
1926a73881c9STrond Myklebust transport->file = filp;
1927a73881c9STrond Myklebust
192822f79326SPavel Emelyanov return sock;
192922f79326SPavel Emelyanov out:
193022f79326SPavel Emelyanov return ERR_PTR(err);
193122f79326SPavel Emelyanov }
193222f79326SPavel Emelyanov
xs_local_finish_connecting(struct rpc_xprt * xprt,struct socket * sock)1933176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt,
1934176e21eeSChuck Lever struct socket *sock)
1935176e21eeSChuck Lever {
1936176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
1937176e21eeSChuck Lever xprt);
1938176e21eeSChuck Lever
1939176e21eeSChuck Lever if (!transport->inet) {
1940176e21eeSChuck Lever struct sock *sk = sock->sk;
1941176e21eeSChuck Lever
1942ea9afca8STrond Myklebust lock_sock(sk);
1943176e21eeSChuck Lever
1944176e21eeSChuck Lever xs_save_old_callbacks(transport, sk);
1945176e21eeSChuck Lever
1946176e21eeSChuck Lever sk->sk_user_data = xprt;
1947a2648094STrond Myklebust sk->sk_data_ready = xs_data_ready;
1948176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space;
1949efce2d0bSTrond Myklebust sk->sk_state_change = xs_local_state_change;
19502118071dSTrond Myklebust sk->sk_error_report = xs_error_report;
195198123866SBenjamin Coddington sk->sk_use_task_frag = false;
1952176e21eeSChuck Lever
1953176e21eeSChuck Lever xprt_clear_connected(xprt);
1954176e21eeSChuck Lever
1955176e21eeSChuck Lever /* Reset to new socket */
1956176e21eeSChuck Lever transport->sock = sock;
1957176e21eeSChuck Lever transport->inet = sk;
1958176e21eeSChuck Lever
1959ea9afca8STrond Myklebust release_sock(sk);
1960176e21eeSChuck Lever }
1961176e21eeSChuck Lever
1962ae053551STrond Myklebust xs_stream_start_connect(transport);
19636c7a64e5STrond Myklebust
1964176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0);
1965176e21eeSChuck Lever }
1966176e21eeSChuck Lever
1967176e21eeSChuck Lever /**
1968176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint
1969176e21eeSChuck Lever * @transport: socket transport to connect
1970176e21eeSChuck Lever */
xs_local_setup_socket(struct sock_xprt * transport)1971dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport)
1972176e21eeSChuck Lever {
1973176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt;
1974a73881c9STrond Myklebust struct file *filp;
1975176e21eeSChuck Lever struct socket *sock;
197668e9a246SColin Ian King int status;
1977176e21eeSChuck Lever
1978176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL,
1979176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1);
1980176e21eeSChuck Lever if (status < 0) {
1981176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL "
1982176e21eeSChuck Lever "transport socket (%d).\n", -status);
1983176e21eeSChuck Lever goto out;
1984176e21eeSChuck Lever }
1985d1358917SStefan Hajnoczi xs_reclassify_socket(AF_LOCAL, sock);
1986176e21eeSChuck Lever
1987a73881c9STrond Myklebust filp = sock_alloc_file(sock, O_NONBLOCK, NULL);
1988a73881c9STrond Myklebust if (IS_ERR(filp)) {
1989a73881c9STrond Myklebust status = PTR_ERR(filp);
1990a73881c9STrond Myklebust goto out;
1991a73881c9STrond Myklebust }
1992a73881c9STrond Myklebust transport->file = filp;
1993a73881c9STrond Myklebust
1994176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n",
1995176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
1996176e21eeSChuck Lever
1997176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock);
199840b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status);
1999176e21eeSChuck Lever switch (status) {
2000176e21eeSChuck Lever case 0:
2001176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n",
2002176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
20033968a8a5SChuck Lever xprt->stat.connect_count++;
20043968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies -
20053968a8a5SChuck Lever xprt->stat.connect_start;
2006176e21eeSChuck Lever xprt_set_connected(xprt);
200793f479d3SGustavo A. R. Silva break;
20083601c4a9STrond Myklebust case -ENOBUFS:
2009176e21eeSChuck Lever break;
2010176e21eeSChuck Lever case -ENOENT:
2011176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n",
2012176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
2013176e21eeSChuck Lever break;
20144a20a988STrond Myklebust case -ECONNREFUSED:
20154a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n",
20164a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]);
20174a20a988STrond Myklebust break;
2018176e21eeSChuck Lever default:
2019176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n",
2020176e21eeSChuck Lever __func__, -status,
2021176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]);
2022176e21eeSChuck Lever }
2023176e21eeSChuck Lever
2024176e21eeSChuck Lever out:
2025176e21eeSChuck Lever xprt_clear_connecting(xprt);
2026176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status);
2027dc107402SJ. Bruce Fields return status;
2028dc107402SJ. Bruce Fields }
2029dc107402SJ. Bruce Fields
xs_local_connect(struct rpc_xprt * xprt,struct rpc_task * task)2030b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task)
2031dc107402SJ. Bruce Fields {
2032dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2033dc107402SJ. Bruce Fields int ret;
2034dc107402SJ. Bruce Fields
2035aad41a7dSTrond Myklebust if (transport->file)
2036aad41a7dSTrond Myklebust goto force_disconnect;
2037aad41a7dSTrond Myklebust
2038dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) {
2039dc107402SJ. Bruce Fields /*
2040dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the
2041dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc
2042dc107402SJ. Bruce Fields * call. Thus we connect synchronously.
2043dc107402SJ. Bruce Fields *
2044dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls,
2045dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to
2046dc107402SJ. Bruce Fields * connect.
2047dc107402SJ. Bruce Fields */
204839494194STrond Myklebust rpc_task_set_rpc_status(task, -ENOTCONN);
2049aad41a7dSTrond Myklebust goto out_wake;
2050dc107402SJ. Bruce Fields }
2051dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport);
2052dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task))
2053dc107402SJ. Bruce Fields msleep_interruptible(15000);
2054aad41a7dSTrond Myklebust return;
2055aad41a7dSTrond Myklebust force_disconnect:
2056aad41a7dSTrond Myklebust xprt_force_disconnect(xprt);
2057aad41a7dSTrond Myklebust out_wake:
2058aad41a7dSTrond Myklebust xprt_clear_connecting(xprt);
2059aad41a7dSTrond Myklebust xprt_wake_pending_tasks(xprt, -ENOTCONN);
2060176e21eeSChuck Lever }
2061176e21eeSChuck Lever
20623c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP)
2063d6e971d8SJeff Layton /*
2064693486d5SNeilBrown * Note that this should be called with XPRT_LOCKED held, or recv_mutex
2065693486d5SNeilBrown * held, or when we otherwise know that we have exclusive access to the
2066693486d5SNeilBrown * socket, to guard against races with xs_reset_transport.
2067d6e971d8SJeff Layton */
xs_set_memalloc(struct rpc_xprt * xprt)2068a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt)
2069a564b8f0SMel Gorman {
2070a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt,
2071a564b8f0SMel Gorman xprt);
2072a564b8f0SMel Gorman
2073d6e971d8SJeff Layton /*
2074d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The
2075d6e971d8SJeff Layton * reconnecting process will get it for us.
2076d6e971d8SJeff Layton */
2077d6e971d8SJeff Layton if (!transport->inet)
2078d6e971d8SJeff Layton return;
20798e228133SJeff Layton if (atomic_read(&xprt->swapper))
2080a564b8f0SMel Gorman sk_set_memalloc(transport->inet);
2081a564b8f0SMel Gorman }
2082a564b8f0SMel Gorman
2083a564b8f0SMel Gorman /**
2084d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap.
2085a564b8f0SMel Gorman * @xprt: transport to tag
2086a564b8f0SMel Gorman *
20878e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and
20888e228133SJeff Layton * optionally mark it for swapping if it wasn't already.
2089a564b8f0SMel Gorman */
2090d67fa4d8SJeff Layton static int
xs_enable_swap(struct rpc_xprt * xprt)2091d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt)
2092a564b8f0SMel Gorman {
2093d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
2094a564b8f0SMel Gorman
2095693486d5SNeilBrown mutex_lock(&xs->recv_mutex);
2096693486d5SNeilBrown if (atomic_inc_return(&xprt->swapper) == 1 &&
2097693486d5SNeilBrown xs->inet)
2098d6e971d8SJeff Layton sk_set_memalloc(xs->inet);
2099693486d5SNeilBrown mutex_unlock(&xs->recv_mutex);
21008e228133SJeff Layton return 0;
2101a564b8f0SMel Gorman }
2102a564b8f0SMel Gorman
21038e228133SJeff Layton /**
2104d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap.
21058e228133SJeff Layton * @xprt: transport to tag
21068e228133SJeff Layton *
21078e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the
21088e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket.
21098e228133SJeff Layton */
2110d67fa4d8SJeff Layton static void
xs_disable_swap(struct rpc_xprt * xprt)2111d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt)
21128e228133SJeff Layton {
2113d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt);
21148e228133SJeff Layton
2115693486d5SNeilBrown mutex_lock(&xs->recv_mutex);
2116693486d5SNeilBrown if (atomic_dec_and_test(&xprt->swapper) &&
2117693486d5SNeilBrown xs->inet)
2118d6e971d8SJeff Layton sk_clear_memalloc(xs->inet);
2119693486d5SNeilBrown mutex_unlock(&xs->recv_mutex);
2120a564b8f0SMel Gorman }
2121a564b8f0SMel Gorman #else
xs_set_memalloc(struct rpc_xprt * xprt)2122a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt)
2123a564b8f0SMel Gorman {
2124a564b8f0SMel Gorman }
2125d67fa4d8SJeff Layton
2126d67fa4d8SJeff Layton static int
xs_enable_swap(struct rpc_xprt * xprt)2127d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt)
2128d67fa4d8SJeff Layton {
2129d67fa4d8SJeff Layton return -EINVAL;
2130d67fa4d8SJeff Layton }
2131d67fa4d8SJeff Layton
2132d67fa4d8SJeff Layton static void
xs_disable_swap(struct rpc_xprt * xprt)2133d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt)
2134d67fa4d8SJeff Layton {
2135d67fa4d8SJeff Layton }
2136a564b8f0SMel Gorman #endif
2137a564b8f0SMel Gorman
xs_udp_finish_connecting(struct rpc_xprt * xprt,struct socket * sock)213816be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
2139a246b010SChuck Lever {
214016be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2141edb267a6SChuck Lever
2142ee0ac0c2SChuck Lever if (!transport->inet) {
2143b0d93ad5SChuck Lever struct sock *sk = sock->sk;
2144b0d93ad5SChuck Lever
2145ea9afca8STrond Myklebust lock_sock(sk);
2146b0d93ad5SChuck Lever
21472a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk);
21482a9e1cfaSTrond Myklebust
2149b0d93ad5SChuck Lever sk->sk_user_data = xprt;
2150f9b2ee71STrond Myklebust sk->sk_data_ready = xs_data_ready;
2151b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space;
215298123866SBenjamin Coddington sk->sk_use_task_frag = false;
2153b0d93ad5SChuck Lever
2154b0d93ad5SChuck Lever xprt_set_connected(xprt);
2155b0d93ad5SChuck Lever
2156b0d93ad5SChuck Lever /* Reset to new socket */
2157ee0ac0c2SChuck Lever transport->sock = sock;
2158ee0ac0c2SChuck Lever transport->inet = sk;
2159b0d93ad5SChuck Lever
2160a564b8f0SMel Gorman xs_set_memalloc(xprt);
2161a564b8f0SMel Gorman
2162ea9afca8STrond Myklebust release_sock(sk);
2163b0d93ad5SChuck Lever }
2164470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt);
216502910177STrond Myklebust
216602910177STrond Myklebust xprt->stat.connect_start = jiffies;
216716be2d20SChuck Lever }
216816be2d20SChuck Lever
xs_udp_setup_socket(struct work_struct * work)21698c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work)
2170a246b010SChuck Lever {
2171a246b010SChuck Lever struct sock_xprt *transport =
2172a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work);
2173a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt;
2174d099b8afSColin Ian King struct socket *sock;
2175b65c0310SPavel Emelyanov int status = -EIO;
21768db55a03SNeilBrown unsigned int pflags = current->flags;
2177a246b010SChuck Lever
21788db55a03SNeilBrown if (atomic_read(&xprt->swapper))
21798db55a03SNeilBrown current->flags |= PF_MEMALLOC;
21808c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport,
21814dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM,
21824dda9c8aSTrond Myklebust IPPROTO_UDP, false);
2183b65c0310SPavel Emelyanov if (IS_ERR(sock))
2184a246b010SChuck Lever goto out;
218568e220bdSChuck Lever
2186c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to "
2187c740eff8SChuck Lever "%s (port %s)\n", xprt,
2188c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO],
2189c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
2190c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]);
219168e220bdSChuck Lever
219268e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock);
219340b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0);
2194a246b010SChuck Lever status = 0;
2195b0d93ad5SChuck Lever out:
2196b0d93ad5SChuck Lever xprt_clear_connecting(xprt);
2197cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport);
21987d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status);
21998db55a03SNeilBrown current_restore_flags(pflags, PF_MEMALLOC);
2200b0d93ad5SChuck Lever }
2201b0d93ad5SChuck Lever
22024876cc77STrond Myklebust /**
22034876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket
22044876cc77STrond Myklebust * @xprt: transport
22054876cc77STrond Myklebust *
22064876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the
22074876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR);
22084876cc77STrond Myklebust */
xs_tcp_shutdown(struct rpc_xprt * xprt)22094876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt)
22104876cc77STrond Myklebust {
22114876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
22124876cc77STrond Myklebust struct socket *sock = transport->sock;
22139b30889cSTrond Myklebust int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE;
22144876cc77STrond Myklebust
22154876cc77STrond Myklebust if (sock == NULL)
22164876cc77STrond Myklebust return;
22170a6ff58eSTrond Myklebust if (!xprt->reuseport) {
22180a6ff58eSTrond Myklebust xs_close(xprt);
22190a6ff58eSTrond Myklebust return;
22200a6ff58eSTrond Myklebust }
22219b30889cSTrond Myklebust switch (skst) {
22227c81e6a9STrond Myklebust case TCP_FIN_WAIT1:
22237c81e6a9STrond Myklebust case TCP_FIN_WAIT2:
2224943d045aSSiddharth Kawar case TCP_LAST_ACK:
22257c81e6a9STrond Myklebust break;
22267c81e6a9STrond Myklebust case TCP_ESTABLISHED:
22277c81e6a9STrond Myklebust case TCP_CLOSE_WAIT:
22284876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR);
22294876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock);
22309b30889cSTrond Myklebust break;
22317c81e6a9STrond Myklebust default:
22324876cc77STrond Myklebust xs_reset_transport(transport);
22334876cc77STrond Myklebust }
22349b30889cSTrond Myklebust }
22354876cc77STrond Myklebust
xs_tcp_set_socket_timeouts(struct rpc_xprt * xprt,struct socket * sock)22368d1b8c62STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt,
22378d1b8c62STrond Myklebust struct socket *sock)
2238b0d93ad5SChuck Lever {
223916be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
22403a107f07STrond Myklebust struct net *net = sock_net(sock->sk);
22413a107f07STrond Myklebust unsigned long connect_timeout;
22423a107f07STrond Myklebust unsigned long syn_retries;
22437196dbb0STrond Myklebust unsigned int keepidle;
22447196dbb0STrond Myklebust unsigned int keepcnt;
2245775f06abSTrond Myklebust unsigned int timeo;
22463a107f07STrond Myklebust unsigned long t;
22477f260e85STrond Myklebust
2248b5e92419STrond Myklebust spin_lock(&xprt->transport_lock);
22497196dbb0STrond Myklebust keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ);
22507196dbb0STrond Myklebust keepcnt = xprt->timeout->to_retries + 1;
22517196dbb0STrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) *
22527196dbb0STrond Myklebust (xprt->timeout->to_retries + 1);
22537196dbb0STrond Myklebust clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
2254b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock);
22557f260e85STrond Myklebust
22567f260e85STrond Myklebust /* TCP Keepalive options */
2257ce3d9544SChristoph Hellwig sock_set_keepalive(sock->sk);
225871c48eb8SChristoph Hellwig tcp_sock_set_keepidle(sock->sk, keepidle);
2259d41ecaacSChristoph Hellwig tcp_sock_set_keepintvl(sock->sk, keepidle);
2260480aeb96SChristoph Hellwig tcp_sock_set_keepcnt(sock->sk, keepcnt);
2261b0d93ad5SChuck Lever
22628d1b8c62STrond Myklebust /* TCP user timeout (see RFC5482) */
2263c488aeadSChristoph Hellwig tcp_sock_set_user_timeout(sock->sk, timeo);
22643a107f07STrond Myklebust
22653a107f07STrond Myklebust /* Connect timeout */
22663a107f07STrond Myklebust connect_timeout = max_t(unsigned long,
22673a107f07STrond Myklebust DIV_ROUND_UP(xprt->connect_timeout, HZ), 1);
22683a107f07STrond Myklebust syn_retries = max_t(unsigned long,
22693a107f07STrond Myklebust READ_ONCE(net->ipv4.sysctl_tcp_syn_retries), 1);
22703a107f07STrond Myklebust for (t = 0; t <= syn_retries && (1UL << t) < connect_timeout; t++)
22713a107f07STrond Myklebust ;
22723a107f07STrond Myklebust if (t <= syn_retries)
22733a107f07STrond Myklebust tcp_sock_set_syncnt(sock->sk, t - 1);
22748d1b8c62STrond Myklebust }
22758d1b8c62STrond Myklebust
xs_tcp_do_set_connect_timeout(struct rpc_xprt * xprt,unsigned long connect_timeout)22763e6ff89dSTrond Myklebust static void xs_tcp_do_set_connect_timeout(struct rpc_xprt *xprt,
22773e6ff89dSTrond Myklebust unsigned long connect_timeout)
22783e6ff89dSTrond Myklebust {
22793e6ff89dSTrond Myklebust struct sock_xprt *transport =
22803e6ff89dSTrond Myklebust container_of(xprt, struct sock_xprt, xprt);
22813e6ff89dSTrond Myklebust struct rpc_timeout to;
22823e6ff89dSTrond Myklebust unsigned long initval;
22833e6ff89dSTrond Myklebust
22843e6ff89dSTrond Myklebust memcpy(&to, xprt->timeout, sizeof(to));
22853e6ff89dSTrond Myklebust /* Arbitrary lower limit */
22863e6ff89dSTrond Myklebust initval = max_t(unsigned long, connect_timeout, XS_TCP_INIT_REEST_TO);
22873e6ff89dSTrond Myklebust to.to_initval = initval;
22883e6ff89dSTrond Myklebust to.to_maxval = initval;
22893e6ff89dSTrond Myklebust to.to_retries = 0;
22903e6ff89dSTrond Myklebust memcpy(&transport->tcp_timeout, &to, sizeof(transport->tcp_timeout));
22913e6ff89dSTrond Myklebust xprt->timeout = &transport->tcp_timeout;
22923e6ff89dSTrond Myklebust xprt->connect_timeout = connect_timeout;
22938d1b8c62STrond Myklebust }
22948d1b8c62STrond Myklebust
xs_tcp_set_connect_timeout(struct rpc_xprt * xprt,unsigned long connect_timeout,unsigned long reconnect_timeout)22957196dbb0STrond Myklebust static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt,
22967196dbb0STrond Myklebust unsigned long connect_timeout,
22977196dbb0STrond Myklebust unsigned long reconnect_timeout)
22987196dbb0STrond Myklebust {
22997196dbb0STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
23007196dbb0STrond Myklebust
2301b5e92419STrond Myklebust spin_lock(&xprt->transport_lock);
23027196dbb0STrond Myklebust if (reconnect_timeout < xprt->max_reconnect_timeout)
23037196dbb0STrond Myklebust xprt->max_reconnect_timeout = reconnect_timeout;
23043e6ff89dSTrond Myklebust if (connect_timeout < xprt->connect_timeout)
23053e6ff89dSTrond Myklebust xs_tcp_do_set_connect_timeout(xprt, connect_timeout);
23067196dbb0STrond Myklebust set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state);
2307b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock);
23087196dbb0STrond Myklebust }
23097196dbb0STrond Myklebust
xs_tcp_finish_connecting(struct rpc_xprt * xprt,struct socket * sock)23108d1b8c62STrond Myklebust static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
23118d1b8c62STrond Myklebust {
23128d1b8c62STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
23138d1b8c62STrond Myklebust
23148d1b8c62STrond Myklebust if (!transport->inet) {
23158d1b8c62STrond Myklebust struct sock *sk = sock->sk;
23168d1b8c62STrond Myklebust
2317d88e4d82SNeilBrown /* Avoid temporary address, they are bad for long-lived
2318d88e4d82SNeilBrown * connections such as NFS mounts.
2319d88e4d82SNeilBrown * RFC4941, section 3.6 suggests that:
2320d88e4d82SNeilBrown * Individual applications, which have specific
2321d88e4d82SNeilBrown * knowledge about the normal duration of connections,
2322d88e4d82SNeilBrown * MAY override this as appropriate.
2323d88e4d82SNeilBrown */
232418d5ad62SChristoph Hellwig if (xs_addr(xprt)->sa_family == PF_INET6) {
232518d5ad62SChristoph Hellwig ip6_sock_set_addr_preferences(sk,
232618d5ad62SChristoph Hellwig IPV6_PREFER_SRC_PUBLIC);
232718d5ad62SChristoph Hellwig }
2328d88e4d82SNeilBrown
23298d1b8c62STrond Myklebust xs_tcp_set_socket_timeouts(xprt, sock);
2330d737e5d4STrond Myklebust tcp_sock_set_nodelay(sk);
2331775f06abSTrond Myklebust
2332ea9afca8STrond Myklebust lock_sock(sk);
2333b0d93ad5SChuck Lever
23342a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk);
23352a9e1cfaSTrond Myklebust
2336b0d93ad5SChuck Lever sk->sk_user_data = xprt;
23375157b956STrond Myklebust sk->sk_data_ready = xs_data_ready;
2338b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change;
2339b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space;
23402118071dSTrond Myklebust sk->sk_error_report = xs_error_report;
234198123866SBenjamin Coddington sk->sk_use_task_frag = false;
23423167e12cSChuck Lever
23433167e12cSChuck Lever /* socket options */
23443167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER);
2345b0d93ad5SChuck Lever
2346b0d93ad5SChuck Lever xprt_clear_connected(xprt);
2347b0d93ad5SChuck Lever
2348b0d93ad5SChuck Lever /* Reset to new socket */
2349ee0ac0c2SChuck Lever transport->sock = sock;
2350ee0ac0c2SChuck Lever transport->inet = sk;
2351b0d93ad5SChuck Lever
2352ea9afca8STrond Myklebust release_sock(sk);
2353b0d93ad5SChuck Lever }
2354b0d93ad5SChuck Lever
235501d37c42STrond Myklebust if (!xprt_bound(xprt))
2356280254b6STrond Myklebust return -ENOTCONN;
235701d37c42STrond Myklebust
2358a564b8f0SMel Gorman xs_set_memalloc(xprt);
2359a564b8f0SMel Gorman
2360ae053551STrond Myklebust xs_stream_start_connect(transport);
2361e1806c7bSTrond Myklebust
2362b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */
23630fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state);
2364280254b6STrond Myklebust return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK);
236516be2d20SChuck Lever }
236616be2d20SChuck Lever
236716be2d20SChuck Lever /**
2368b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint
2369acf0a39fSChuck Lever * @work: queued work item
237016be2d20SChuck Lever *
237116be2d20SChuck Lever * Invoked by a work queue tasklet.
237216be2d20SChuck Lever */
xs_tcp_setup_socket(struct work_struct * work)2373cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work)
237416be2d20SChuck Lever {
2375cdd518d5SPavel Emelyanov struct sock_xprt *transport =
2376cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work);
237716be2d20SChuck Lever struct socket *sock = transport->sock;
2378a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt;
2379280254b6STrond Myklebust int status;
23808db55a03SNeilBrown unsigned int pflags = current->flags;
238116be2d20SChuck Lever
23828db55a03SNeilBrown if (atomic_read(&xprt->swapper))
23838db55a03SNeilBrown current->flags |= PF_MEMALLOC;
238489f42494STrond Myklebust
238589f42494STrond Myklebust if (xprt_connected(xprt))
238689f42494STrond Myklebust goto out;
238789f42494STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECT_SENT,
238889f42494STrond Myklebust &transport->sock_state) ||
238989f42494STrond Myklebust !sock) {
239089f42494STrond Myklebust xs_reset_transport(transport);
239189f42494STrond Myklebust sock = xs_create_sock(xprt, transport, xs_addr(xprt)->sa_family,
239289f42494STrond Myklebust SOCK_STREAM, IPPROTO_TCP, true);
2393b61d59ffSTrond Myklebust if (IS_ERR(sock)) {
2394280254b6STrond Myklebust xprt_wake_pending_tasks(xprt, PTR_ERR(sock));
239516be2d20SChuck Lever goto out;
239616be2d20SChuck Lever }
23977d1e8255STrond Myklebust }
23987d1e8255STrond Myklebust
2399c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to "
2400c740eff8SChuck Lever "%s (port %s)\n", xprt,
2401c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO],
2402c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
2403c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]);
240416be2d20SChuck Lever
240516be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock);
240640b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status);
2407a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n",
240846121cf7SChuck Lever xprt, -status, xprt_connected(xprt),
240946121cf7SChuck Lever sock->sk->sk_state);
2410a246b010SChuck Lever switch (status) {
24112a491991STrond Myklebust case 0:
2412a246b010SChuck Lever case -EINPROGRESS:
2413280254b6STrond Myklebust /* SYN_SENT! */
241489f42494STrond Myklebust set_bit(XPRT_SOCK_CONNECT_SENT, &transport->sock_state);
2415280254b6STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO)
2416280254b6STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
2417280254b6STrond Myklebust fallthrough;
2418a246b010SChuck Lever case -EALREADY:
2419280254b6STrond Myklebust goto out_unlock;
2420280254b6STrond Myklebust case -EADDRNOTAVAIL:
2421280254b6STrond Myklebust /* Source port number is unavailable. Try a new one! */
2422280254b6STrond Myklebust transport->srcport = 0;
2423280254b6STrond Myklebust status = -EAGAIN;
2424280254b6STrond Myklebust break;
24259fcfe0c8STrond Myklebust case -EINVAL:
24269fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link
24279fcfe0c8STrond Myklebust * local IPv6 address without a scope-id.
24289fcfe0c8STrond Myklebust */
24293ed5e2a2STrond Myklebust case -ECONNREFUSED:
24303ed5e2a2STrond Myklebust case -ECONNRESET:
2431eb5b46faSTrond Myklebust case -ENETDOWN:
24323ed5e2a2STrond Myklebust case -ENETUNREACH:
24334ba161a7STrond Myklebust case -EHOSTUNREACH:
24343913c78cSTrond Myklebust case -EADDRINUSE:
24353601c4a9STrond Myklebust case -ENOBUFS:
2436280254b6STrond Myklebust break;
2437280254b6STrond Myklebust default:
2438280254b6STrond Myklebust printk("%s: connect returned unhandled error %d\n",
2439280254b6STrond Myklebust __func__, status);
2440280254b6STrond Myklebust status = -EAGAIN;
2441280254b6STrond Myklebust }
2442280254b6STrond Myklebust
24438c71139dSCalum Mackay /* xs_tcp_force_close() wakes tasks with a fixed error code.
24448c71139dSCalum Mackay * We need to wake them first to ensure the correct error code.
24456ea44adcSNeilBrown */
24466ea44adcSNeilBrown xprt_wake_pending_tasks(xprt, status);
24474efdd92cSTrond Myklebust xs_tcp_force_close(xprt);
2448a246b010SChuck Lever out:
24492226feb6SChuck Lever xprt_clear_connecting(xprt);
2450280254b6STrond Myklebust out_unlock:
2451cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport);
24528db55a03SNeilBrown current_restore_flags(pflags, PF_MEMALLOC);
2453a246b010SChuck Lever }
2454a246b010SChuck Lever
245575eb6af7SChuck Lever /*
245675eb6af7SChuck Lever * Transfer the connected socket to @upper_transport, then mark that
245775eb6af7SChuck Lever * xprt CONNECTED.
245875eb6af7SChuck Lever */
xs_tcp_tls_finish_connecting(struct rpc_xprt * lower_xprt,struct sock_xprt * upper_transport)245975eb6af7SChuck Lever static int xs_tcp_tls_finish_connecting(struct rpc_xprt *lower_xprt,
246075eb6af7SChuck Lever struct sock_xprt *upper_transport)
246175eb6af7SChuck Lever {
246275eb6af7SChuck Lever struct sock_xprt *lower_transport =
246375eb6af7SChuck Lever container_of(lower_xprt, struct sock_xprt, xprt);
246475eb6af7SChuck Lever struct rpc_xprt *upper_xprt = &upper_transport->xprt;
246575eb6af7SChuck Lever
246675eb6af7SChuck Lever if (!upper_transport->inet) {
246775eb6af7SChuck Lever struct socket *sock = lower_transport->sock;
246875eb6af7SChuck Lever struct sock *sk = sock->sk;
246975eb6af7SChuck Lever
247075eb6af7SChuck Lever /* Avoid temporary address, they are bad for long-lived
247175eb6af7SChuck Lever * connections such as NFS mounts.
247275eb6af7SChuck Lever * RFC4941, section 3.6 suggests that:
247375eb6af7SChuck Lever * Individual applications, which have specific
247475eb6af7SChuck Lever * knowledge about the normal duration of connections,
247575eb6af7SChuck Lever * MAY override this as appropriate.
247675eb6af7SChuck Lever */
247775eb6af7SChuck Lever if (xs_addr(upper_xprt)->sa_family == PF_INET6)
247875eb6af7SChuck Lever ip6_sock_set_addr_preferences(sk, IPV6_PREFER_SRC_PUBLIC);
247975eb6af7SChuck Lever
248075eb6af7SChuck Lever xs_tcp_set_socket_timeouts(upper_xprt, sock);
248175eb6af7SChuck Lever tcp_sock_set_nodelay(sk);
248275eb6af7SChuck Lever
248375eb6af7SChuck Lever lock_sock(sk);
248475eb6af7SChuck Lever
248575eb6af7SChuck Lever /* @sk is already connected, so it now has the RPC callbacks.
248675eb6af7SChuck Lever * Reach into @lower_transport to save the original ones.
248775eb6af7SChuck Lever */
248875eb6af7SChuck Lever upper_transport->old_data_ready = lower_transport->old_data_ready;
248975eb6af7SChuck Lever upper_transport->old_state_change = lower_transport->old_state_change;
249075eb6af7SChuck Lever upper_transport->old_write_space = lower_transport->old_write_space;
249175eb6af7SChuck Lever upper_transport->old_error_report = lower_transport->old_error_report;
249275eb6af7SChuck Lever sk->sk_user_data = upper_xprt;
249375eb6af7SChuck Lever
249475eb6af7SChuck Lever /* socket options */
249575eb6af7SChuck Lever sock_reset_flag(sk, SOCK_LINGER);
249675eb6af7SChuck Lever
249775eb6af7SChuck Lever xprt_clear_connected(upper_xprt);
249875eb6af7SChuck Lever
249975eb6af7SChuck Lever upper_transport->sock = sock;
250075eb6af7SChuck Lever upper_transport->inet = sk;
250175eb6af7SChuck Lever upper_transport->file = lower_transport->file;
250275eb6af7SChuck Lever
250375eb6af7SChuck Lever release_sock(sk);
250475eb6af7SChuck Lever
250575eb6af7SChuck Lever /* Reset lower_transport before shutting down its clnt */
250675eb6af7SChuck Lever mutex_lock(&lower_transport->recv_mutex);
250775eb6af7SChuck Lever lower_transport->inet = NULL;
250875eb6af7SChuck Lever lower_transport->sock = NULL;
250975eb6af7SChuck Lever lower_transport->file = NULL;
251075eb6af7SChuck Lever
251175eb6af7SChuck Lever xprt_clear_connected(lower_xprt);
251275eb6af7SChuck Lever xs_sock_reset_connection_flags(lower_xprt);
251375eb6af7SChuck Lever xs_stream_reset_connect(lower_transport);
251475eb6af7SChuck Lever mutex_unlock(&lower_transport->recv_mutex);
251575eb6af7SChuck Lever }
251675eb6af7SChuck Lever
251775eb6af7SChuck Lever if (!xprt_bound(upper_xprt))
251875eb6af7SChuck Lever return -ENOTCONN;
251975eb6af7SChuck Lever
252075eb6af7SChuck Lever xs_set_memalloc(upper_xprt);
252175eb6af7SChuck Lever
252275eb6af7SChuck Lever if (!xprt_test_and_set_connected(upper_xprt)) {
252375eb6af7SChuck Lever upper_xprt->connect_cookie++;
252475eb6af7SChuck Lever clear_bit(XPRT_SOCK_CONNECTING, &upper_transport->sock_state);
252575eb6af7SChuck Lever xprt_clear_connecting(upper_xprt);
252675eb6af7SChuck Lever
252775eb6af7SChuck Lever upper_xprt->stat.connect_count++;
252875eb6af7SChuck Lever upper_xprt->stat.connect_time += (long)jiffies -
252975eb6af7SChuck Lever upper_xprt->stat.connect_start;
253075eb6af7SChuck Lever xs_run_error_worker(upper_transport, XPRT_SOCK_WAKE_PENDING);
253175eb6af7SChuck Lever }
253275eb6af7SChuck Lever return 0;
253375eb6af7SChuck Lever }
253475eb6af7SChuck Lever
253575eb6af7SChuck Lever /**
253675eb6af7SChuck Lever * xs_tls_handshake_done - TLS handshake completion handler
253775eb6af7SChuck Lever * @data: address of xprt to wake
253875eb6af7SChuck Lever * @status: status of handshake
253975eb6af7SChuck Lever * @peerid: serial number of key containing the remote's identity
254075eb6af7SChuck Lever *
254175eb6af7SChuck Lever */
xs_tls_handshake_done(void * data,int status,key_serial_t peerid)254275eb6af7SChuck Lever static void xs_tls_handshake_done(void *data, int status, key_serial_t peerid)
254375eb6af7SChuck Lever {
254475eb6af7SChuck Lever struct rpc_xprt *lower_xprt = data;
254575eb6af7SChuck Lever struct sock_xprt *lower_transport =
254675eb6af7SChuck Lever container_of(lower_xprt, struct sock_xprt, xprt);
254775eb6af7SChuck Lever
254875eb6af7SChuck Lever lower_transport->xprt_err = status ? -EACCES : 0;
254975eb6af7SChuck Lever complete(&lower_transport->handshake_done);
255075eb6af7SChuck Lever xprt_put(lower_xprt);
255175eb6af7SChuck Lever }
255275eb6af7SChuck Lever
xs_tls_handshake_sync(struct rpc_xprt * lower_xprt,struct xprtsec_parms * xprtsec)255375eb6af7SChuck Lever static int xs_tls_handshake_sync(struct rpc_xprt *lower_xprt, struct xprtsec_parms *xprtsec)
255475eb6af7SChuck Lever {
255575eb6af7SChuck Lever struct sock_xprt *lower_transport =
255675eb6af7SChuck Lever container_of(lower_xprt, struct sock_xprt, xprt);
255775eb6af7SChuck Lever struct tls_handshake_args args = {
255875eb6af7SChuck Lever .ta_sock = lower_transport->sock,
255975eb6af7SChuck Lever .ta_done = xs_tls_handshake_done,
256075eb6af7SChuck Lever .ta_data = xprt_get(lower_xprt),
256175eb6af7SChuck Lever .ta_peername = lower_xprt->servername,
256275eb6af7SChuck Lever };
256375eb6af7SChuck Lever struct sock *sk = lower_transport->inet;
256475eb6af7SChuck Lever int rc;
256575eb6af7SChuck Lever
256675eb6af7SChuck Lever init_completion(&lower_transport->handshake_done);
256775eb6af7SChuck Lever set_bit(XPRT_SOCK_IGNORE_RECV, &lower_transport->sock_state);
256875eb6af7SChuck Lever lower_transport->xprt_err = -ETIMEDOUT;
256975eb6af7SChuck Lever switch (xprtsec->policy) {
257075eb6af7SChuck Lever case RPC_XPRTSEC_TLS_ANON:
257175eb6af7SChuck Lever rc = tls_client_hello_anon(&args, GFP_KERNEL);
257275eb6af7SChuck Lever if (rc)
257375eb6af7SChuck Lever goto out_put_xprt;
257475eb6af7SChuck Lever break;
257575eb6af7SChuck Lever case RPC_XPRTSEC_TLS_X509:
257675eb6af7SChuck Lever args.ta_my_cert = xprtsec->cert_serial;
257775eb6af7SChuck Lever args.ta_my_privkey = xprtsec->privkey_serial;
257875eb6af7SChuck Lever rc = tls_client_hello_x509(&args, GFP_KERNEL);
257975eb6af7SChuck Lever if (rc)
258075eb6af7SChuck Lever goto out_put_xprt;
258175eb6af7SChuck Lever break;
258275eb6af7SChuck Lever default:
258375eb6af7SChuck Lever rc = -EACCES;
258475eb6af7SChuck Lever goto out_put_xprt;
258575eb6af7SChuck Lever }
258675eb6af7SChuck Lever
258775eb6af7SChuck Lever rc = wait_for_completion_interruptible_timeout(&lower_transport->handshake_done,
258875eb6af7SChuck Lever XS_TLS_HANDSHAKE_TO);
258975eb6af7SChuck Lever if (rc <= 0) {
259075eb6af7SChuck Lever if (!tls_handshake_cancel(sk)) {
259175eb6af7SChuck Lever if (rc == 0)
259275eb6af7SChuck Lever rc = -ETIMEDOUT;
259375eb6af7SChuck Lever goto out_put_xprt;
259475eb6af7SChuck Lever }
259575eb6af7SChuck Lever }
259675eb6af7SChuck Lever
259775eb6af7SChuck Lever rc = lower_transport->xprt_err;
259875eb6af7SChuck Lever
259975eb6af7SChuck Lever out:
260075eb6af7SChuck Lever xs_stream_reset_connect(lower_transport);
260175eb6af7SChuck Lever clear_bit(XPRT_SOCK_IGNORE_RECV, &lower_transport->sock_state);
260275eb6af7SChuck Lever return rc;
260375eb6af7SChuck Lever
260475eb6af7SChuck Lever out_put_xprt:
260575eb6af7SChuck Lever xprt_put(lower_xprt);
260675eb6af7SChuck Lever goto out;
260775eb6af7SChuck Lever }
260875eb6af7SChuck Lever
260975eb6af7SChuck Lever /**
261075eb6af7SChuck Lever * xs_tcp_tls_setup_socket - establish a TLS session on a TCP socket
261175eb6af7SChuck Lever * @work: queued work item
261275eb6af7SChuck Lever *
261375eb6af7SChuck Lever * Invoked by a work queue tasklet.
261475eb6af7SChuck Lever *
261575eb6af7SChuck Lever * For RPC-with-TLS, there is a two-stage connection process.
261675eb6af7SChuck Lever *
261775eb6af7SChuck Lever * The "upper-layer xprt" is visible to the RPC consumer. Once it has
261875eb6af7SChuck Lever * been marked connected, the consumer knows that a TCP connection and
261975eb6af7SChuck Lever * a TLS session have been established.
262075eb6af7SChuck Lever *
262175eb6af7SChuck Lever * A "lower-layer xprt", created in this function, handles the mechanics
262275eb6af7SChuck Lever * of connecting the TCP socket, performing the RPC_AUTH_TLS probe, and
262375eb6af7SChuck Lever * then driving the TLS handshake. Once all that is complete, the upper
262475eb6af7SChuck Lever * layer xprt is marked connected.
262575eb6af7SChuck Lever */
xs_tcp_tls_setup_socket(struct work_struct * work)262675eb6af7SChuck Lever static void xs_tcp_tls_setup_socket(struct work_struct *work)
262775eb6af7SChuck Lever {
262875eb6af7SChuck Lever struct sock_xprt *upper_transport =
262975eb6af7SChuck Lever container_of(work, struct sock_xprt, connect_worker.work);
263075eb6af7SChuck Lever struct rpc_clnt *upper_clnt = upper_transport->clnt;
263175eb6af7SChuck Lever struct rpc_xprt *upper_xprt = &upper_transport->xprt;
263275eb6af7SChuck Lever struct rpc_create_args args = {
263375eb6af7SChuck Lever .net = upper_xprt->xprt_net,
263475eb6af7SChuck Lever .protocol = upper_xprt->prot,
263575eb6af7SChuck Lever .address = (struct sockaddr *)&upper_xprt->addr,
263675eb6af7SChuck Lever .addrsize = upper_xprt->addrlen,
263775eb6af7SChuck Lever .timeout = upper_clnt->cl_timeout,
263875eb6af7SChuck Lever .servername = upper_xprt->servername,
263975eb6af7SChuck Lever .program = upper_clnt->cl_program,
264075eb6af7SChuck Lever .prognumber = upper_clnt->cl_prog,
264175eb6af7SChuck Lever .version = upper_clnt->cl_vers,
264275eb6af7SChuck Lever .authflavor = RPC_AUTH_TLS,
264375eb6af7SChuck Lever .cred = upper_clnt->cl_cred,
264475eb6af7SChuck Lever .xprtsec = {
264575eb6af7SChuck Lever .policy = RPC_XPRTSEC_NONE,
264675eb6af7SChuck Lever },
26479b332c72SOlga Kornievskaia .stats = upper_clnt->cl_stats,
264875eb6af7SChuck Lever };
264975eb6af7SChuck Lever unsigned int pflags = current->flags;
265075eb6af7SChuck Lever struct rpc_clnt *lower_clnt;
265175eb6af7SChuck Lever struct rpc_xprt *lower_xprt;
265275eb6af7SChuck Lever int status;
265375eb6af7SChuck Lever
265475eb6af7SChuck Lever if (atomic_read(&upper_xprt->swapper))
265575eb6af7SChuck Lever current->flags |= PF_MEMALLOC;
265675eb6af7SChuck Lever
265775eb6af7SChuck Lever xs_stream_start_connect(upper_transport);
265875eb6af7SChuck Lever
265975eb6af7SChuck Lever /* This implicitly sends an RPC_AUTH_TLS probe */
266075eb6af7SChuck Lever lower_clnt = rpc_create(&args);
266175eb6af7SChuck Lever if (IS_ERR(lower_clnt)) {
266275eb6af7SChuck Lever trace_rpc_tls_unavailable(upper_clnt, upper_xprt);
266375eb6af7SChuck Lever clear_bit(XPRT_SOCK_CONNECTING, &upper_transport->sock_state);
266475eb6af7SChuck Lever xprt_clear_connecting(upper_xprt);
266575eb6af7SChuck Lever xprt_wake_pending_tasks(upper_xprt, PTR_ERR(lower_clnt));
266675eb6af7SChuck Lever xs_run_error_worker(upper_transport, XPRT_SOCK_WAKE_PENDING);
266775eb6af7SChuck Lever goto out_unlock;
266875eb6af7SChuck Lever }
266975eb6af7SChuck Lever
267075eb6af7SChuck Lever /* RPC_AUTH_TLS probe was successful. Try a TLS handshake on
267175eb6af7SChuck Lever * the lower xprt.
267275eb6af7SChuck Lever */
267375eb6af7SChuck Lever rcu_read_lock();
267475eb6af7SChuck Lever lower_xprt = rcu_dereference(lower_clnt->cl_xprt);
267575eb6af7SChuck Lever rcu_read_unlock();
267626e8bfa3SAnna Schumaker
267726e8bfa3SAnna Schumaker if (wait_on_bit_lock(&lower_xprt->state, XPRT_LOCKED, TASK_KILLABLE))
267826e8bfa3SAnna Schumaker goto out_unlock;
267926e8bfa3SAnna Schumaker
268075eb6af7SChuck Lever status = xs_tls_handshake_sync(lower_xprt, &upper_xprt->xprtsec);
268175eb6af7SChuck Lever if (status) {
268275eb6af7SChuck Lever trace_rpc_tls_not_started(upper_clnt, upper_xprt);
268375eb6af7SChuck Lever goto out_close;
268475eb6af7SChuck Lever }
268575eb6af7SChuck Lever
268675eb6af7SChuck Lever status = xs_tcp_tls_finish_connecting(lower_xprt, upper_transport);
268775eb6af7SChuck Lever if (status)
268875eb6af7SChuck Lever goto out_close;
268926e8bfa3SAnna Schumaker xprt_release_write(lower_xprt, NULL);
269075eb6af7SChuck Lever
269175eb6af7SChuck Lever trace_rpc_socket_connect(upper_xprt, upper_transport->sock, 0);
269275eb6af7SChuck Lever if (!xprt_test_and_set_connected(upper_xprt)) {
269375eb6af7SChuck Lever upper_xprt->connect_cookie++;
269475eb6af7SChuck Lever clear_bit(XPRT_SOCK_CONNECTING, &upper_transport->sock_state);
269575eb6af7SChuck Lever xprt_clear_connecting(upper_xprt);
269675eb6af7SChuck Lever
269775eb6af7SChuck Lever upper_xprt->stat.connect_count++;
269875eb6af7SChuck Lever upper_xprt->stat.connect_time += (long)jiffies -
269975eb6af7SChuck Lever upper_xprt->stat.connect_start;
270075eb6af7SChuck Lever xs_run_error_worker(upper_transport, XPRT_SOCK_WAKE_PENDING);
270175eb6af7SChuck Lever }
270275eb6af7SChuck Lever rpc_shutdown_client(lower_clnt);
270375eb6af7SChuck Lever
270475eb6af7SChuck Lever out_unlock:
270575eb6af7SChuck Lever current_restore_flags(pflags, PF_MEMALLOC);
270675eb6af7SChuck Lever upper_transport->clnt = NULL;
270775eb6af7SChuck Lever xprt_unlock_connect(upper_xprt, upper_transport);
270875eb6af7SChuck Lever return;
270975eb6af7SChuck Lever
271075eb6af7SChuck Lever out_close:
271126e8bfa3SAnna Schumaker xprt_release_write(lower_xprt, NULL);
271275eb6af7SChuck Lever rpc_shutdown_client(lower_clnt);
271375eb6af7SChuck Lever
271475eb6af7SChuck Lever /* xprt_force_disconnect() wakes tasks with a fixed tk_status code.
271575eb6af7SChuck Lever * Wake them first here to ensure they get our tk_status code.
271675eb6af7SChuck Lever */
271775eb6af7SChuck Lever xprt_wake_pending_tasks(upper_xprt, status);
271875eb6af7SChuck Lever xs_tcp_force_close(upper_xprt);
271975eb6af7SChuck Lever xprt_clear_connecting(upper_xprt);
272075eb6af7SChuck Lever goto out_unlock;
272175eb6af7SChuck Lever }
272275eb6af7SChuck Lever
272368e220bdSChuck Lever /**
27249903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint
27251b092092STrond Myklebust * @xprt: pointer to transport structure
27269903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request
27279903cd1cSChuck Lever *
27289903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting.
272903bf4b70SChuck Lever *
273003bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway
273103bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a
273203bf4b70SChuck Lever * socket on a privileged port.
273303bf4b70SChuck Lever *
273403bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents
273503bf4b70SChuck Lever * retry floods (hard mounts).
27369903cd1cSChuck Lever */
xs_connect(struct rpc_xprt * xprt,struct rpc_task * task)27371b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task)
2738a246b010SChuck Lever {
2739ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
274002910177STrond Myklebust unsigned long delay = 0;
2741a246b010SChuck Lever
2742718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport));
2743718ba5b8STrond Myklebust
274489f42494STrond Myklebust if (transport->sock != NULL) {
274546121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu "
274689f42494STrond Myklebust "seconds\n", xprt, xprt->reestablish_timeout / HZ);
274799b1a4c3STrond Myklebust
2748675dd90aSChuck Lever delay = xprt_reconnect_delay(xprt);
2749675dd90aSChuck Lever xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO);
275002910177STrond Myklebust
275102910177STrond Myklebust } else
27529903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt);
275302910177STrond Myklebust
275475eb6af7SChuck Lever transport->clnt = task->tk_client;
275540a5f1b1STrond Myklebust queue_delayed_work(xprtiod_workqueue,
275602910177STrond Myklebust &transport->connect_worker,
275702910177STrond Myklebust delay);
2758a246b010SChuck Lever }
2759a246b010SChuck Lever
xs_wake_disconnect(struct sock_xprt * transport)27604f8943f8STrond Myklebust static void xs_wake_disconnect(struct sock_xprt *transport)
27614f8943f8STrond Myklebust {
27624f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state))
27634f8943f8STrond Myklebust xs_tcp_force_close(&transport->xprt);
27644f8943f8STrond Myklebust }
27654f8943f8STrond Myklebust
xs_wake_write(struct sock_xprt * transport)27664f8943f8STrond Myklebust static void xs_wake_write(struct sock_xprt *transport)
27674f8943f8STrond Myklebust {
27684f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state))
27694f8943f8STrond Myklebust xprt_write_space(&transport->xprt);
27704f8943f8STrond Myklebust }
27714f8943f8STrond Myklebust
xs_wake_error(struct sock_xprt * transport)27724f8943f8STrond Myklebust static void xs_wake_error(struct sock_xprt *transport)
27734f8943f8STrond Myklebust {
27744f8943f8STrond Myklebust int sockerr;
27754f8943f8STrond Myklebust
27764f8943f8STrond Myklebust if (!test_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
27774f8943f8STrond Myklebust return;
27784f8943f8STrond Myklebust mutex_lock(&transport->recv_mutex);
27794f8943f8STrond Myklebust if (transport->sock == NULL)
27804f8943f8STrond Myklebust goto out;
27814f8943f8STrond Myklebust if (!test_and_clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state))
27824f8943f8STrond Myklebust goto out;
2783af84537dSBenjamin Coddington sockerr = xchg(&transport->xprt_err, 0);
27844f8943f8STrond Myklebust if (sockerr < 0)
27854f8943f8STrond Myklebust xprt_wake_pending_tasks(&transport->xprt, sockerr);
27864f8943f8STrond Myklebust out:
27874f8943f8STrond Myklebust mutex_unlock(&transport->recv_mutex);
27884f8943f8STrond Myklebust }
27894f8943f8STrond Myklebust
xs_wake_pending(struct sock_xprt * transport)27904f8943f8STrond Myklebust static void xs_wake_pending(struct sock_xprt *transport)
27914f8943f8STrond Myklebust {
27924f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_PENDING, &transport->sock_state))
27934f8943f8STrond Myklebust xprt_wake_pending_tasks(&transport->xprt, -EAGAIN);
27944f8943f8STrond Myklebust }
27954f8943f8STrond Myklebust
xs_error_handle(struct work_struct * work)27964f8943f8STrond Myklebust static void xs_error_handle(struct work_struct *work)
27974f8943f8STrond Myklebust {
27984f8943f8STrond Myklebust struct sock_xprt *transport = container_of(work,
27994f8943f8STrond Myklebust struct sock_xprt, error_worker);
28004f8943f8STrond Myklebust
28014f8943f8STrond Myklebust xs_wake_disconnect(transport);
28024f8943f8STrond Myklebust xs_wake_write(transport);
28034f8943f8STrond Myklebust xs_wake_error(transport);
28044f8943f8STrond Myklebust xs_wake_pending(transport);
28054f8943f8STrond Myklebust }
28064f8943f8STrond Myklebust
2807262ca07dSChuck Lever /**
280812b20ce3SBhaskar Chowdhury * xs_local_print_stats - display AF_LOCAL socket-specific stats
2809176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics
2810176e21eeSChuck Lever * @seq: output file
2811176e21eeSChuck Lever *
2812176e21eeSChuck Lever */
xs_local_print_stats(struct rpc_xprt * xprt,struct seq_file * seq)2813176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2814176e21eeSChuck Lever {
2815176e21eeSChuck Lever long idle_time = 0;
2816176e21eeSChuck Lever
2817176e21eeSChuck Lever if (xprt_connected(xprt))
2818176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ;
2819176e21eeSChuck Lever
2820176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu "
282115a45206SAndy Adamson "%llu %llu %lu %llu %llu\n",
2822176e21eeSChuck Lever xprt->stat.bind_count,
2823176e21eeSChuck Lever xprt->stat.connect_count,
28248440a886SChuck Lever xprt->stat.connect_time / HZ,
2825176e21eeSChuck Lever idle_time,
2826176e21eeSChuck Lever xprt->stat.sends,
2827176e21eeSChuck Lever xprt->stat.recvs,
2828176e21eeSChuck Lever xprt->stat.bad_xids,
2829176e21eeSChuck Lever xprt->stat.req_u,
283015a45206SAndy Adamson xprt->stat.bklog_u,
283115a45206SAndy Adamson xprt->stat.max_slots,
283215a45206SAndy Adamson xprt->stat.sending_u,
283315a45206SAndy Adamson xprt->stat.pending_u);
2834176e21eeSChuck Lever }
2835176e21eeSChuck Lever
2836176e21eeSChuck Lever /**
283712b20ce3SBhaskar Chowdhury * xs_udp_print_stats - display UDP socket-specific stats
2838262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics
2839262ca07dSChuck Lever * @seq: output file
2840262ca07dSChuck Lever *
2841262ca07dSChuck Lever */
xs_udp_print_stats(struct rpc_xprt * xprt,struct seq_file * seq)2842262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2843262ca07dSChuck Lever {
2844c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2845c8475461SChuck Lever
284615a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu "
284715a45206SAndy Adamson "%lu %llu %llu\n",
2848fbfffbd5SChuck Lever transport->srcport,
2849262ca07dSChuck Lever xprt->stat.bind_count,
2850262ca07dSChuck Lever xprt->stat.sends,
2851262ca07dSChuck Lever xprt->stat.recvs,
2852262ca07dSChuck Lever xprt->stat.bad_xids,
2853262ca07dSChuck Lever xprt->stat.req_u,
285415a45206SAndy Adamson xprt->stat.bklog_u,
285515a45206SAndy Adamson xprt->stat.max_slots,
285615a45206SAndy Adamson xprt->stat.sending_u,
285715a45206SAndy Adamson xprt->stat.pending_u);
2858262ca07dSChuck Lever }
2859262ca07dSChuck Lever
2860262ca07dSChuck Lever /**
286112b20ce3SBhaskar Chowdhury * xs_tcp_print_stats - display TCP socket-specific stats
2862262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics
2863262ca07dSChuck Lever * @seq: output file
2864262ca07dSChuck Lever *
2865262ca07dSChuck Lever */
xs_tcp_print_stats(struct rpc_xprt * xprt,struct seq_file * seq)2866262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq)
2867262ca07dSChuck Lever {
2868c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt);
2869262ca07dSChuck Lever long idle_time = 0;
2870262ca07dSChuck Lever
2871262ca07dSChuck Lever if (xprt_connected(xprt))
2872262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ;
2873262ca07dSChuck Lever
287415a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu "
287515a45206SAndy Adamson "%llu %llu %lu %llu %llu\n",
2876fbfffbd5SChuck Lever transport->srcport,
2877262ca07dSChuck Lever xprt->stat.bind_count,
2878262ca07dSChuck Lever xprt->stat.connect_count,
28798440a886SChuck Lever xprt->stat.connect_time / HZ,
2880262ca07dSChuck Lever idle_time,
2881262ca07dSChuck Lever xprt->stat.sends,
2882262ca07dSChuck Lever xprt->stat.recvs,
2883262ca07dSChuck Lever xprt->stat.bad_xids,
2884262ca07dSChuck Lever xprt->stat.req_u,
288515a45206SAndy Adamson xprt->stat.bklog_u,
288615a45206SAndy Adamson xprt->stat.max_slots,
288715a45206SAndy Adamson xprt->stat.sending_u,
288815a45206SAndy Adamson xprt->stat.pending_u);
2889262ca07dSChuck Lever }
2890262ca07dSChuck Lever
28914cfc7e60SRahul Iyer /*
28924cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason
28934cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want
28944cfc7e60SRahul Iyer * to use the server side send routines.
28954cfc7e60SRahul Iyer */
bc_malloc(struct rpc_task * task)28965fe6eaa1SChuck Lever static int bc_malloc(struct rpc_task *task)
28974cfc7e60SRahul Iyer {
28985fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp;
28995fe6eaa1SChuck Lever size_t size = rqst->rq_callsize;
29004cfc7e60SRahul Iyer struct page *page;
29014cfc7e60SRahul Iyer struct rpc_buffer *buf;
29024cfc7e60SRahul Iyer
29035fe6eaa1SChuck Lever if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) {
29045fe6eaa1SChuck Lever WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n",
29055fe6eaa1SChuck Lever size);
29065fe6eaa1SChuck Lever return -EINVAL;
29075fe6eaa1SChuck Lever }
29084cfc7e60SRahul Iyer
2909b2648015STrond Myklebust page = alloc_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
29104cfc7e60SRahul Iyer if (!page)
29115fe6eaa1SChuck Lever return -ENOMEM;
29124cfc7e60SRahul Iyer
29134cfc7e60SRahul Iyer buf = page_address(page);
29144cfc7e60SRahul Iyer buf->len = PAGE_SIZE;
29154cfc7e60SRahul Iyer
29165fe6eaa1SChuck Lever rqst->rq_buffer = buf->data;
291718e601d6SJeff Layton rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize;
29185fe6eaa1SChuck Lever return 0;
29194cfc7e60SRahul Iyer }
29204cfc7e60SRahul Iyer
29214cfc7e60SRahul Iyer /*
29224cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine
29234cfc7e60SRahul Iyer */
bc_free(struct rpc_task * task)29243435c74aSChuck Lever static void bc_free(struct rpc_task *task)
29254cfc7e60SRahul Iyer {
29263435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer;
29274cfc7e60SRahul Iyer struct rpc_buffer *buf;
29284cfc7e60SRahul Iyer
29294cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data);
29304cfc7e60SRahul Iyer free_page((unsigned long)buf);
29314cfc7e60SRahul Iyer }
29324cfc7e60SRahul Iyer
bc_sendto(struct rpc_rqst * req)29334cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req)
29344cfc7e60SRahul Iyer {
2935da1661b9SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf;
29364cfc7e60SRahul Iyer struct sock_xprt *transport =
2937067fb11bSChuck Lever container_of(req->rq_xprt, struct sock_xprt, xprt);
2938067fb11bSChuck Lever struct msghdr msg = {
2939da1661b9SChuck Lever .msg_flags = 0,
2940067fb11bSChuck Lever };
2941067fb11bSChuck Lever rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT |
2942da1661b9SChuck Lever (u32)xdr->len);
2943da1661b9SChuck Lever unsigned int sent = 0;
2944da1661b9SChuck Lever int err;
29454cfc7e60SRahul Iyer
29468729aabaSChuck Lever req->rq_xtime = ktime_get();
2947ff053dbbSTrond Myklebust err = xdr_alloc_bvec(xdr, rpc_task_gfp_mask());
2948ff053dbbSTrond Myklebust if (err < 0)
2949ff053dbbSTrond Myklebust return err;
2950da1661b9SChuck Lever err = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, marker, &sent);
2951da1661b9SChuck Lever xdr_free_bvec(xdr);
2952da1661b9SChuck Lever if (err < 0 || sent != (xdr->len + sizeof(marker)))
2953067fb11bSChuck Lever return -EAGAIN;
2954da1661b9SChuck Lever return sent;
29554cfc7e60SRahul Iyer }
29564cfc7e60SRahul Iyer
2957ca4faf54SChuck Lever /**
2958ca4faf54SChuck Lever * bc_send_request - Send a backchannel Call on a TCP socket
2959ca4faf54SChuck Lever * @req: rpc_rqst containing Call message to be sent
2960ca4faf54SChuck Lever *
2961ca4faf54SChuck Lever * xpt_mutex ensures @rqstp's whole message is written to the socket
2962ca4faf54SChuck Lever * without interruption.
2963ca4faf54SChuck Lever *
2964ca4faf54SChuck Lever * Return values:
2965ca4faf54SChuck Lever * %0 if the message was sent successfully
2966ca4faf54SChuck Lever * %ENOTCONN if the message was not sent
29674cfc7e60SRahul Iyer */
bc_send_request(struct rpc_rqst * req)2968adfa7144STrond Myklebust static int bc_send_request(struct rpc_rqst *req)
29694cfc7e60SRahul Iyer {
29704cfc7e60SRahul Iyer struct svc_xprt *xprt;
29717fc56136SAndrzej Hajda int len;
29724cfc7e60SRahul Iyer
29734cfc7e60SRahul Iyer /*
29744cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt
29754cfc7e60SRahul Iyer */
29764cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt;
29774cfc7e60SRahul Iyer
29784cfc7e60SRahul Iyer /*
29794cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared
29804cfc7e60SRahul Iyer * with the fore channel
29814cfc7e60SRahul Iyer */
2982c544577dSTrond Myklebust mutex_lock(&xprt->xpt_mutex);
29834cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags))
29844cfc7e60SRahul Iyer len = -ENOTCONN;
29854cfc7e60SRahul Iyer else
29864cfc7e60SRahul Iyer len = bc_sendto(req);
29874cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex);
29884cfc7e60SRahul Iyer
29894cfc7e60SRahul Iyer if (len > 0)
29904cfc7e60SRahul Iyer len = 0;
29914cfc7e60SRahul Iyer
29924cfc7e60SRahul Iyer return len;
29934cfc7e60SRahul Iyer }
29944cfc7e60SRahul Iyer
29954cfc7e60SRahul Iyer /*
29964cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing
29974cfc7e60SRahul Iyer */
29984cfc7e60SRahul Iyer
bc_close(struct rpc_xprt * xprt)29994cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt)
30004cfc7e60SRahul Iyer {
30016221f1d9SChuck Lever xprt_disconnect_done(xprt);
30024cfc7e60SRahul Iyer }
30034cfc7e60SRahul Iyer
30044cfc7e60SRahul Iyer /*
30054cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client
30064cfc7e60SRahul Iyer * initiated, we do nothing
30074cfc7e60SRahul Iyer */
30084cfc7e60SRahul Iyer
bc_destroy(struct rpc_xprt * xprt)30094cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt)
30104cfc7e60SRahul Iyer {
301147f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt);
301247f72efaSKinglong Mee
301347f72efaSKinglong Mee xs_xprt_free(xprt);
301447f72efaSKinglong Mee module_put(THIS_MODULE);
30154cfc7e60SRahul Iyer }
30164cfc7e60SRahul Iyer
3017d31ae254SChuck Lever static const struct rpc_xprt_ops xs_local_ops = {
3018176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt,
30194cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt,
3020f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot,
3021a9cde23aSChuck Lever .free_slot = xprt_free_slot,
3022176e21eeSChuck Lever .rpcbind = xs_local_rpcbind,
3023176e21eeSChuck Lever .set_port = xs_local_set_port,
3024dc107402SJ. Bruce Fields .connect = xs_local_connect,
3025176e21eeSChuck Lever .buf_alloc = rpc_malloc,
3026176e21eeSChuck Lever .buf_free = rpc_free,
3027550aebfeSTrond Myklebust .prepare_request = xs_stream_prepare_request,
3028176e21eeSChuck Lever .send_request = xs_local_send_request,
30298ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def,
3030176e21eeSChuck Lever .close = xs_close,
3031a1311d87STrond Myklebust .destroy = xs_destroy,
3032176e21eeSChuck Lever .print_stats = xs_local_print_stats,
3033d67fa4d8SJeff Layton .enable_swap = xs_enable_swap,
3034d67fa4d8SJeff Layton .disable_swap = xs_disable_swap,
3035176e21eeSChuck Lever };
3036176e21eeSChuck Lever
3037d31ae254SChuck Lever static const struct rpc_xprt_ops xs_udp_ops = {
303843118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size,
303912a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong,
304049e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong,
3041f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot,
3042a9cde23aSChuck Lever .free_slot = xprt_free_slot,
304345160d62SChuck Lever .rpcbind = rpcb_getport_async,
304492200412SChuck Lever .set_port = xs_set_port,
30459903cd1cSChuck Lever .connect = xs_connect,
3046421ab1beSTrond Myklebust .get_srcaddr = xs_sock_srcaddr,
3047421ab1beSTrond Myklebust .get_srcport = xs_sock_srcport,
304802107148SChuck Lever .buf_alloc = rpc_malloc,
304902107148SChuck Lever .buf_free = rpc_free,
3050262965f5SChuck Lever .send_request = xs_udp_send_request,
30518ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_rtt,
305246c0ee8bSChuck Lever .timer = xs_udp_timer,
3053a58dd398SChuck Lever .release_request = xprt_release_rqst_cong,
3054262965f5SChuck Lever .close = xs_close,
3055262965f5SChuck Lever .destroy = xs_destroy,
3056262ca07dSChuck Lever .print_stats = xs_udp_print_stats,
3057d67fa4d8SJeff Layton .enable_swap = xs_enable_swap,
3058d67fa4d8SJeff Layton .disable_swap = xs_disable_swap,
30594a068258SChuck Lever .inject_disconnect = xs_inject_disconnect,
3060262965f5SChuck Lever };
3061262965f5SChuck Lever
3062d31ae254SChuck Lever static const struct rpc_xprt_ops xs_tcp_ops = {
306312a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt,
30644cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt,
306536bd7de9STrond Myklebust .alloc_slot = xprt_alloc_slot,
3066a9cde23aSChuck Lever .free_slot = xprt_free_slot,
306745160d62SChuck Lever .rpcbind = rpcb_getport_async,
306892200412SChuck Lever .set_port = xs_set_port,
30690b9e7943STrond Myklebust .connect = xs_connect,
3070421ab1beSTrond Myklebust .get_srcaddr = xs_sock_srcaddr,
3071421ab1beSTrond Myklebust .get_srcport = xs_sock_srcport,
307202107148SChuck Lever .buf_alloc = rpc_malloc,
307302107148SChuck Lever .buf_free = rpc_free,
3074277e4ab7STrond Myklebust .prepare_request = xs_stream_prepare_request,
3075262965f5SChuck Lever .send_request = xs_tcp_send_request,
30768ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def,
3077c627d31bSTrond Myklebust .close = xs_tcp_shutdown,
30789903cd1cSChuck Lever .destroy = xs_destroy,
30797196dbb0STrond Myklebust .set_connect_timeout = xs_tcp_set_connect_timeout,
3080262ca07dSChuck Lever .print_stats = xs_tcp_print_stats,
3081d67fa4d8SJeff Layton .enable_swap = xs_enable_swap,
3082d67fa4d8SJeff Layton .disable_swap = xs_disable_swap,
30834a068258SChuck Lever .inject_disconnect = xs_inject_disconnect,
308442e5c3e2SChuck Lever #ifdef CONFIG_SUNRPC_BACKCHANNEL
308542e5c3e2SChuck Lever .bc_setup = xprt_setup_bc,
30866b26cc8cSChuck Lever .bc_maxpayload = xs_tcp_bc_maxpayload,
30877402a4feSTrond Myklebust .bc_num_slots = xprt_bc_max_slots,
308842e5c3e2SChuck Lever .bc_free_rqst = xprt_free_bc_rqst,
308942e5c3e2SChuck Lever .bc_destroy = xprt_destroy_bc,
309042e5c3e2SChuck Lever #endif
3091a246b010SChuck Lever };
3092a246b010SChuck Lever
30934cfc7e60SRahul Iyer /*
30944cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel
30954cfc7e60SRahul Iyer */
30964cfc7e60SRahul Iyer
3097d31ae254SChuck Lever static const struct rpc_xprt_ops bc_tcp_ops = {
30984cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt,
30994cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt,
310084e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot,
3101a9cde23aSChuck Lever .free_slot = xprt_free_slot,
31024cfc7e60SRahul Iyer .buf_alloc = bc_malloc,
31034cfc7e60SRahul Iyer .buf_free = bc_free,
31044cfc7e60SRahul Iyer .send_request = bc_send_request,
31058ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def,
31064cfc7e60SRahul Iyer .close = bc_close,
31074cfc7e60SRahul Iyer .destroy = bc_destroy,
31084cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats,
3109d67fa4d8SJeff Layton .enable_swap = xs_enable_swap,
3110d67fa4d8SJeff Layton .disable_swap = xs_disable_swap,
31114a068258SChuck Lever .inject_disconnect = xs_inject_disconnect,
31124cfc7e60SRahul Iyer };
31134cfc7e60SRahul Iyer
xs_init_anyaddr(const int family,struct sockaddr * sap)311492476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap)
311592476850SChuck Lever {
311692476850SChuck Lever static const struct sockaddr_in sin = {
311792476850SChuck Lever .sin_family = AF_INET,
311892476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY),
311992476850SChuck Lever };
312092476850SChuck Lever static const struct sockaddr_in6 sin6 = {
312192476850SChuck Lever .sin6_family = AF_INET6,
312292476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT,
312392476850SChuck Lever };
312492476850SChuck Lever
312592476850SChuck Lever switch (family) {
3126176e21eeSChuck Lever case AF_LOCAL:
3127176e21eeSChuck Lever break;
312892476850SChuck Lever case AF_INET:
312992476850SChuck Lever memcpy(sap, &sin, sizeof(sin));
313092476850SChuck Lever break;
313192476850SChuck Lever case AF_INET6:
313292476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6));
313392476850SChuck Lever break;
313492476850SChuck Lever default:
313592476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__);
313692476850SChuck Lever return -EAFNOSUPPORT;
313792476850SChuck Lever }
313892476850SChuck Lever return 0;
313992476850SChuck Lever }
314092476850SChuck Lever
xs_setup_xprt(struct xprt_create * args,unsigned int slot_table_size,unsigned int max_slot_table_size)31413c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args,
3142d9ba131dSTrond Myklebust unsigned int slot_table_size,
3143d9ba131dSTrond Myklebust unsigned int max_slot_table_size)
3144c8541ecdSChuck Lever {
3145c8541ecdSChuck Lever struct rpc_xprt *xprt;
3146ffc2e518SChuck Lever struct sock_xprt *new;
3147c8541ecdSChuck Lever
314896802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) {
3149c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n");
3150c8541ecdSChuck Lever return ERR_PTR(-EBADF);
3151c8541ecdSChuck Lever }
3152c8541ecdSChuck Lever
3153d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size,
3154d9ba131dSTrond Myklebust max_slot_table_size);
3155bd1722d4SPavel Emelyanov if (xprt == NULL) {
315646121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate "
315746121cf7SChuck Lever "rpc_xprt\n");
3158c8541ecdSChuck Lever return ERR_PTR(-ENOMEM);
3159c8541ecdSChuck Lever }
3160c8541ecdSChuck Lever
3161bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt);
3162edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex);
316396802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen);
316496802a09SFrank van Maarseveen xprt->addrlen = args->addrlen;
3165d3bc9a1dSFrank van Maarseveen if (args->srcaddr)
3166fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen);
316792476850SChuck Lever else {
316892476850SChuck Lever int err;
316992476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family,
317092476850SChuck Lever (struct sockaddr *)&new->srcaddr);
31712aa13531SStanislav Kinsbursky if (err != 0) {
31722aa13531SStanislav Kinsbursky xprt_free(xprt);
317392476850SChuck Lever return ERR_PTR(err);
317492476850SChuck Lever }
31752aa13531SStanislav Kinsbursky }
3176c8541ecdSChuck Lever
3177c8541ecdSChuck Lever return xprt;
3178c8541ecdSChuck Lever }
3179c8541ecdSChuck Lever
3180176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = {
3181176e21eeSChuck Lever .to_initval = 10 * HZ,
3182176e21eeSChuck Lever .to_maxval = 10 * HZ,
3183176e21eeSChuck Lever .to_retries = 2,
3184176e21eeSChuck Lever };
3185176e21eeSChuck Lever
3186176e21eeSChuck Lever /**
3187176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket
3188176e21eeSChuck Lever * @args: rpc transport creation arguments
3189176e21eeSChuck Lever *
3190176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP
3191176e21eeSChuck Lever */
xs_setup_local(struct xprt_create * args)3192176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args)
3193176e21eeSChuck Lever {
3194176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr;
3195176e21eeSChuck Lever struct sock_xprt *transport;
3196176e21eeSChuck Lever struct rpc_xprt *xprt;
3197176e21eeSChuck Lever struct rpc_xprt *ret;
3198176e21eeSChuck Lever
3199d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3200d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries);
3201176e21eeSChuck Lever if (IS_ERR(xprt))
3202176e21eeSChuck Lever return xprt;
3203176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt);
3204176e21eeSChuck Lever
3205176e21eeSChuck Lever xprt->prot = 0;
3206d3abc739SOlga Kornievskaia xprt->xprt_class = &xs_local_transport;
3207176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3208176e21eeSChuck Lever
3209176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO;
3210176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
3211176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO;
3212176e21eeSChuck Lever
3213176e21eeSChuck Lever xprt->ops = &xs_local_ops;
3214176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout;
3215176e21eeSChuck Lever
3216550aebfeSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
32174f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle);
3218550aebfeSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket);
321993dc41bdSNeilBrown
3220176e21eeSChuck Lever switch (sun->sun_family) {
3221176e21eeSChuck Lever case AF_LOCAL:
32224388ce05SNeilBrown if (sun->sun_path[0] != '/' && sun->sun_path[0] != '\0') {
3223176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n",
3224176e21eeSChuck Lever sun->sun_path);
3225176e21eeSChuck Lever ret = ERR_PTR(-EINVAL);
3226176e21eeSChuck Lever goto out_err;
3227176e21eeSChuck Lever }
3228176e21eeSChuck Lever xprt_set_bound(xprt);
3229176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL);
3230176e21eeSChuck Lever break;
3231176e21eeSChuck Lever default:
3232176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT);
3233176e21eeSChuck Lever goto out_err;
3234176e21eeSChuck Lever }
3235176e21eeSChuck Lever
3236176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n",
3237176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]);
3238176e21eeSChuck Lever
3239176e21eeSChuck Lever if (try_module_get(THIS_MODULE))
3240176e21eeSChuck Lever return xprt;
3241176e21eeSChuck Lever ret = ERR_PTR(-EINVAL);
3242176e21eeSChuck Lever out_err:
3243315f3812SKinglong Mee xs_xprt_free(xprt);
3244176e21eeSChuck Lever return ret;
3245176e21eeSChuck Lever }
3246176e21eeSChuck Lever
32472881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = {
32482881ae74STrond Myklebust .to_initval = 5 * HZ,
32492881ae74STrond Myklebust .to_maxval = 30 * HZ,
32502881ae74STrond Myklebust .to_increment = 5 * HZ,
32512881ae74STrond Myklebust .to_retries = 5,
32522881ae74STrond Myklebust };
32532881ae74STrond Myklebust
32549903cd1cSChuck Lever /**
32559903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket
325696802a09SFrank van Maarseveen * @args: rpc transport creation arguments
32579903cd1cSChuck Lever *
32589903cd1cSChuck Lever */
xs_setup_udp(struct xprt_create * args)3259483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args)
3260a246b010SChuck Lever {
32618f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr;
3262c8541ecdSChuck Lever struct rpc_xprt *xprt;
3263c8475461SChuck Lever struct sock_xprt *transport;
32640a68b0beSJ. Bruce Fields struct rpc_xprt *ret;
3265a246b010SChuck Lever
3266d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries,
3267d9ba131dSTrond Myklebust xprt_udp_slot_table_entries);
3268c8541ecdSChuck Lever if (IS_ERR(xprt))
3269c8541ecdSChuck Lever return xprt;
3270c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt);
3271a246b010SChuck Lever
3272ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP;
3273d3abc739SOlga Kornievskaia xprt->xprt_class = &xs_udp_transport;
3274a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */
3275a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
3276a246b010SChuck Lever
327703bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO;
327803bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO;
327903bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO;
3280a246b010SChuck Lever
3281262965f5SChuck Lever xprt->ops = &xs_udp_ops;
3282a246b010SChuck Lever
3283ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout;
3284a246b010SChuck Lever
3285f9b2ee71STrond Myklebust INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn);
32864f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle);
3287edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket);
3288edc1b01cSTrond Myklebust
32898f9d5b1aSChuck Lever switch (addr->sa_family) {
32908f9d5b1aSChuck Lever case AF_INET:
32918f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0))
32928f9d5b1aSChuck Lever xprt_set_bound(xprt);
32938f9d5b1aSChuck Lever
32949dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP);
32958f9d5b1aSChuck Lever break;
32968f9d5b1aSChuck Lever case AF_INET6:
32978f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
32988f9d5b1aSChuck Lever xprt_set_bound(xprt);
32998f9d5b1aSChuck Lever
33009dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6);
33018f9d5b1aSChuck Lever break;
33028f9d5b1aSChuck Lever default:
33030a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT);
33040a68b0beSJ. Bruce Fields goto out_err;
33058f9d5b1aSChuck Lever }
33068f9d5b1aSChuck Lever
3307c740eff8SChuck Lever if (xprt_bound(xprt))
3308c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3309c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
3310c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT],
3311c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
3312c740eff8SChuck Lever else
3313c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n",
3314c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
3315c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
3316edb267a6SChuck Lever
3317bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE))
3318c8541ecdSChuck Lever return xprt;
33190a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL);
33200a68b0beSJ. Bruce Fields out_err:
3321315f3812SKinglong Mee xs_xprt_free(xprt);
33220a68b0beSJ. Bruce Fields return ret;
3323a246b010SChuck Lever }
3324a246b010SChuck Lever
33252881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = {
33262881ae74STrond Myklebust .to_initval = 60 * HZ,
33272881ae74STrond Myklebust .to_maxval = 60 * HZ,
33282881ae74STrond Myklebust .to_retries = 2,
33292881ae74STrond Myklebust };
33302881ae74STrond Myklebust
33319903cd1cSChuck Lever /**
33329903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket
333396802a09SFrank van Maarseveen * @args: rpc transport creation arguments
33349903cd1cSChuck Lever *
33359903cd1cSChuck Lever */
xs_setup_tcp(struct xprt_create * args)3336483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args)
3337a246b010SChuck Lever {
33388f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr;
3339c8541ecdSChuck Lever struct rpc_xprt *xprt;
3340c8475461SChuck Lever struct sock_xprt *transport;
33410a68b0beSJ. Bruce Fields struct rpc_xprt *ret;
3342b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;
3343b7993cebSTrond Myklebust
3344b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
3345b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
3346a246b010SChuck Lever
3347d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3348b7993cebSTrond Myklebust max_slot_table_size);
3349c8541ecdSChuck Lever if (IS_ERR(xprt))
3350c8541ecdSChuck Lever return xprt;
3351c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt);
3352a246b010SChuck Lever
3353ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP;
3354d3abc739SOlga Kornievskaia xprt->xprt_class = &xs_tcp_transport;
3355808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3356a246b010SChuck Lever
335703bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO;
335803bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
335903bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO;
3360a246b010SChuck Lever
3361262965f5SChuck Lever xprt->ops = &xs_tcp_ops;
3362ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout;
3363a246b010SChuck Lever
33643851f1cdSTrond Myklebust xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
3365d2ee4138STrond Myklebust if (args->reconnect_timeout)
3366d2ee4138STrond Myklebust xprt->max_reconnect_timeout = args->reconnect_timeout;
3367d2ee4138STrond Myklebust
33687196dbb0STrond Myklebust xprt->connect_timeout = xprt->timeout->to_initval *
33697196dbb0STrond Myklebust (xprt->timeout->to_retries + 1);
3370d2ee4138STrond Myklebust if (args->connect_timeout)
3371d2ee4138STrond Myklebust xs_tcp_do_set_connect_timeout(xprt, args->connect_timeout);
33723851f1cdSTrond Myklebust
3373c50b8ee0STrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
33744f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle);
3375edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket);
3376edc1b01cSTrond Myklebust
33778f9d5b1aSChuck Lever switch (addr->sa_family) {
33788f9d5b1aSChuck Lever case AF_INET:
33798f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0))
33808f9d5b1aSChuck Lever xprt_set_bound(xprt);
33818f9d5b1aSChuck Lever
33829dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
33838f9d5b1aSChuck Lever break;
33848f9d5b1aSChuck Lever case AF_INET6:
33858f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
33868f9d5b1aSChuck Lever xprt_set_bound(xprt);
33878f9d5b1aSChuck Lever
33889dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
33898f9d5b1aSChuck Lever break;
33908f9d5b1aSChuck Lever default:
33910a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT);
33920a68b0beSJ. Bruce Fields goto out_err;
33938f9d5b1aSChuck Lever }
33948f9d5b1aSChuck Lever
3395c740eff8SChuck Lever if (xprt_bound(xprt))
3396c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3397c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
3398c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT],
3399c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
3400c740eff8SChuck Lever else
3401c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n",
3402c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
3403c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
3404c740eff8SChuck Lever
3405bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE))
3406c8541ecdSChuck Lever return xprt;
34070a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL);
34080a68b0beSJ. Bruce Fields out_err:
3409315f3812SKinglong Mee xs_xprt_free(xprt);
34100a68b0beSJ. Bruce Fields return ret;
3411a246b010SChuck Lever }
3412282b32e1SChuck Lever
3413f300babaSAlexandros Batsakis /**
341475eb6af7SChuck Lever * xs_setup_tcp_tls - Set up transport to use a TCP with TLS
341575eb6af7SChuck Lever * @args: rpc transport creation arguments
341675eb6af7SChuck Lever *
341775eb6af7SChuck Lever */
xs_setup_tcp_tls(struct xprt_create * args)341875eb6af7SChuck Lever static struct rpc_xprt *xs_setup_tcp_tls(struct xprt_create *args)
341975eb6af7SChuck Lever {
342075eb6af7SChuck Lever struct sockaddr *addr = args->dstaddr;
342175eb6af7SChuck Lever struct rpc_xprt *xprt;
342275eb6af7SChuck Lever struct sock_xprt *transport;
342375eb6af7SChuck Lever struct rpc_xprt *ret;
342475eb6af7SChuck Lever unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries;
342575eb6af7SChuck Lever
342675eb6af7SChuck Lever if (args->flags & XPRT_CREATE_INFINITE_SLOTS)
342775eb6af7SChuck Lever max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT;
342875eb6af7SChuck Lever
342975eb6af7SChuck Lever xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
343075eb6af7SChuck Lever max_slot_table_size);
343175eb6af7SChuck Lever if (IS_ERR(xprt))
343275eb6af7SChuck Lever return xprt;
343375eb6af7SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt);
343475eb6af7SChuck Lever
343575eb6af7SChuck Lever xprt->prot = IPPROTO_TCP;
343675eb6af7SChuck Lever xprt->xprt_class = &xs_tcp_transport;
343775eb6af7SChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
343875eb6af7SChuck Lever
343975eb6af7SChuck Lever xprt->bind_timeout = XS_BIND_TO;
344075eb6af7SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
344175eb6af7SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO;
344275eb6af7SChuck Lever
344375eb6af7SChuck Lever xprt->ops = &xs_tcp_ops;
344475eb6af7SChuck Lever xprt->timeout = &xs_tcp_default_timeout;
344575eb6af7SChuck Lever
344675eb6af7SChuck Lever xprt->max_reconnect_timeout = xprt->timeout->to_maxval;
344775eb6af7SChuck Lever xprt->connect_timeout = xprt->timeout->to_initval *
344875eb6af7SChuck Lever (xprt->timeout->to_retries + 1);
344975eb6af7SChuck Lever
345075eb6af7SChuck Lever INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn);
345175eb6af7SChuck Lever INIT_WORK(&transport->error_worker, xs_error_handle);
345275eb6af7SChuck Lever
345375eb6af7SChuck Lever switch (args->xprtsec.policy) {
345475eb6af7SChuck Lever case RPC_XPRTSEC_TLS_ANON:
345575eb6af7SChuck Lever case RPC_XPRTSEC_TLS_X509:
345675eb6af7SChuck Lever xprt->xprtsec = args->xprtsec;
345775eb6af7SChuck Lever INIT_DELAYED_WORK(&transport->connect_worker,
345875eb6af7SChuck Lever xs_tcp_tls_setup_socket);
345975eb6af7SChuck Lever break;
346075eb6af7SChuck Lever default:
346175eb6af7SChuck Lever ret = ERR_PTR(-EACCES);
346275eb6af7SChuck Lever goto out_err;
346375eb6af7SChuck Lever }
346475eb6af7SChuck Lever
346575eb6af7SChuck Lever switch (addr->sa_family) {
346675eb6af7SChuck Lever case AF_INET:
346775eb6af7SChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0))
346875eb6af7SChuck Lever xprt_set_bound(xprt);
346975eb6af7SChuck Lever
347075eb6af7SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP);
347175eb6af7SChuck Lever break;
347275eb6af7SChuck Lever case AF_INET6:
347375eb6af7SChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
347475eb6af7SChuck Lever xprt_set_bound(xprt);
347575eb6af7SChuck Lever
347675eb6af7SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6);
347775eb6af7SChuck Lever break;
347875eb6af7SChuck Lever default:
347975eb6af7SChuck Lever ret = ERR_PTR(-EAFNOSUPPORT);
348075eb6af7SChuck Lever goto out_err;
348175eb6af7SChuck Lever }
348275eb6af7SChuck Lever
348375eb6af7SChuck Lever if (xprt_bound(xprt))
348475eb6af7SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n",
348575eb6af7SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
348675eb6af7SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT],
348775eb6af7SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
348875eb6af7SChuck Lever else
348975eb6af7SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n",
349075eb6af7SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR],
349175eb6af7SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]);
349275eb6af7SChuck Lever
349375eb6af7SChuck Lever if (try_module_get(THIS_MODULE))
349475eb6af7SChuck Lever return xprt;
349575eb6af7SChuck Lever ret = ERR_PTR(-EINVAL);
349675eb6af7SChuck Lever out_err:
349775eb6af7SChuck Lever xs_xprt_free(xprt);
349875eb6af7SChuck Lever return ret;
349975eb6af7SChuck Lever }
350075eb6af7SChuck Lever
350175eb6af7SChuck Lever /**
3502f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket
3503f300babaSAlexandros Batsakis * @args: rpc transport creation arguments
3504f300babaSAlexandros Batsakis *
3505f300babaSAlexandros Batsakis */
xs_setup_bc_tcp(struct xprt_create * args)3506f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args)
3507f300babaSAlexandros Batsakis {
3508f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr;
3509f300babaSAlexandros Batsakis struct rpc_xprt *xprt;
3510f300babaSAlexandros Batsakis struct sock_xprt *transport;
3511f300babaSAlexandros Batsakis struct svc_sock *bc_sock;
35120a68b0beSJ. Bruce Fields struct rpc_xprt *ret;
3513f300babaSAlexandros Batsakis
3514d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries,
3515d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries);
3516f300babaSAlexandros Batsakis if (IS_ERR(xprt))
3517f300babaSAlexandros Batsakis return xprt;
3518f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt);
3519f300babaSAlexandros Batsakis
3520f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP;
3521d3abc739SOlga Kornievskaia xprt->xprt_class = &xs_bc_tcp_transport;
3522f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
3523f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout;
3524f300babaSAlexandros Batsakis
3525f300babaSAlexandros Batsakis /* backchannel */
3526f300babaSAlexandros Batsakis xprt_set_bound(xprt);
3527f300babaSAlexandros Batsakis xprt->bind_timeout = 0;
3528f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0;
3529f300babaSAlexandros Batsakis xprt->idle_timeout = 0;
3530f300babaSAlexandros Batsakis
3531f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops;
3532f300babaSAlexandros Batsakis
3533f300babaSAlexandros Batsakis switch (addr->sa_family) {
3534f300babaSAlexandros Batsakis case AF_INET:
3535f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp",
3536f300babaSAlexandros Batsakis RPCBIND_NETID_TCP);
3537f300babaSAlexandros Batsakis break;
3538f300babaSAlexandros Batsakis case AF_INET6:
3539f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp",
3540f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6);
3541f300babaSAlexandros Batsakis break;
3542f300babaSAlexandros Batsakis default:
35430a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT);
35440a68b0beSJ. Bruce Fields goto out_err;
3545f300babaSAlexandros Batsakis }
3546f300babaSAlexandros Batsakis
3547f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n",
3548f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR],
3549f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT],
3550f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]);
3551f300babaSAlexandros Batsakis
3552f300babaSAlexandros Batsakis /*
355399de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection,
355428303ca3SWeng Meiling * we want to keep it around as long as the connection lasts,
355528303ca3SWeng Meiling * in case we need to start using it for a backchannel again;
355628303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed.
355799de8ea9SJ. Bruce Fields */
355899de8ea9SJ. Bruce Fields xprt_get(xprt);
355999de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt;
356099de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt;
356199de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt);
356299de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock;
356399de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk;
356499de8ea9SJ. Bruce Fields
356599de8ea9SJ. Bruce Fields /*
3566f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set
3567f300babaSAlexandros Batsakis * the xprt status to connected
3568f300babaSAlexandros Batsakis */
3569f300babaSAlexandros Batsakis xprt_set_connected(xprt);
3570f300babaSAlexandros Batsakis
3571f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE))
3572f300babaSAlexandros Batsakis return xprt;
3573642aab58SKinglong Mee
3574642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL;
357539a9beabSJ. Bruce Fields args->bc_xprt->xpt_bc_xps = NULL;
357699de8ea9SJ. Bruce Fields xprt_put(xprt);
35770a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL);
35780a68b0beSJ. Bruce Fields out_err:
3579315f3812SKinglong Mee xs_xprt_free(xprt);
35800a68b0beSJ. Bruce Fields return ret;
3581f300babaSAlexandros Batsakis }
3582f300babaSAlexandros Batsakis
3583176e21eeSChuck Lever static struct xprt_class xs_local_transport = {
3584176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list),
3585176e21eeSChuck Lever .name = "named UNIX socket",
3586176e21eeSChuck Lever .owner = THIS_MODULE,
3587176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL,
3588176e21eeSChuck Lever .setup = xs_setup_local,
3589d5aa6b22STrond Myklebust .netid = { "" },
3590176e21eeSChuck Lever };
3591176e21eeSChuck Lever
3592bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = {
3593bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list),
3594bc25571eS\"Talpey, Thomas\ .name = "udp",
3595bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE,
3596f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP,
3597bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp,
3598d5aa6b22STrond Myklebust .netid = { "udp", "udp6", "" },
3599bc25571eS\"Talpey, Thomas\ };
3600bc25571eS\"Talpey, Thomas\
3601bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = {
3602bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list),
3603bc25571eS\"Talpey, Thomas\ .name = "tcp",
3604bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE,
3605f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP,
3606bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp,
3607d5aa6b22STrond Myklebust .netid = { "tcp", "tcp6", "" },
3608bc25571eS\"Talpey, Thomas\ };
3609bc25571eS\"Talpey, Thomas\
361075eb6af7SChuck Lever static struct xprt_class xs_tcp_tls_transport = {
361175eb6af7SChuck Lever .list = LIST_HEAD_INIT(xs_tcp_tls_transport.list),
361275eb6af7SChuck Lever .name = "tcp-with-tls",
361375eb6af7SChuck Lever .owner = THIS_MODULE,
361475eb6af7SChuck Lever .ident = XPRT_TRANSPORT_TCP_TLS,
361575eb6af7SChuck Lever .setup = xs_setup_tcp_tls,
361675eb6af7SChuck Lever .netid = { "tcp", "tcp6", "" },
361775eb6af7SChuck Lever };
361875eb6af7SChuck Lever
3619f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = {
3620f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list),
3621f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel",
3622f300babaSAlexandros Batsakis .owner = THIS_MODULE,
3623f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP,
3624f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp,
3625d5aa6b22STrond Myklebust .netid = { "" },
3626f300babaSAlexandros Batsakis };
3627f300babaSAlexandros Batsakis
3628282b32e1SChuck Lever /**
3629bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client
3630282b32e1SChuck Lever *
3631282b32e1SChuck Lever */
init_socket_xprt(void)3632282b32e1SChuck Lever int init_socket_xprt(void)
3633282b32e1SChuck Lever {
36342b1bec5fSEric W. Biederman if (!sunrpc_table_header)
3635c946cb69SLuis Chamberlain sunrpc_table_header = register_sysctl("sunrpc", xs_tunables_table);
3636fbf76683SChuck Lever
3637176e21eeSChuck Lever xprt_register_transport(&xs_local_transport);
3638bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport);
3639bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport);
364075eb6af7SChuck Lever xprt_register_transport(&xs_tcp_tls_transport);
3641f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport);
3642bc25571eS\"Talpey, Thomas\
3643282b32e1SChuck Lever return 0;
3644282b32e1SChuck Lever }
3645282b32e1SChuck Lever
3646282b32e1SChuck Lever /**
3647bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister
3648282b32e1SChuck Lever *
3649282b32e1SChuck Lever */
cleanup_socket_xprt(void)3650282b32e1SChuck Lever void cleanup_socket_xprt(void)
3651282b32e1SChuck Lever {
3652fbf76683SChuck Lever if (sunrpc_table_header) {
3653fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header);
3654fbf76683SChuck Lever sunrpc_table_header = NULL;
3655fbf76683SChuck Lever }
3656bc25571eS\"Talpey, Thomas\
3657176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport);
3658bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport);
3659bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport);
366075eb6af7SChuck Lever xprt_unregister_transport(&xs_tcp_tls_transport);
3661f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport);
3662282b32e1SChuck Lever }
3663cbf11071STrond Myklebust
param_set_portnr(const char * val,const struct kernel_param * kp)36649bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp)
3665cbf11071STrond Myklebust {
3666cbf11071STrond Myklebust return param_set_uint_minmax(val, kp,
3667cbf11071STrond Myklebust RPC_MIN_RESVPORT,
3668cbf11071STrond Myklebust RPC_MAX_RESVPORT);
3669cbf11071STrond Myklebust }
3670cbf11071STrond Myklebust
36719c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = {
36729bbb9e5aSRusty Russell .set = param_set_portnr,
36739bbb9e5aSRusty Russell .get = param_get_uint,
36749bbb9e5aSRusty Russell };
36759bbb9e5aSRusty Russell
3676cbf11071STrond Myklebust #define param_check_portnr(name, p) \
3677cbf11071STrond Myklebust __param_check(name, p, unsigned int);
3678cbf11071STrond Myklebust
3679cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644);
3680cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644);
3681cbf11071STrond Myklebust
param_set_slot_table_size(const char * val,const struct kernel_param * kp)36829bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val,
36839bbb9e5aSRusty Russell const struct kernel_param *kp)
3684cbf11071STrond Myklebust {
3685cbf11071STrond Myklebust return param_set_uint_minmax(val, kp,
3686cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE,
3687cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE);
3688cbf11071STrond Myklebust }
3689cbf11071STrond Myklebust
36909c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = {
36919bbb9e5aSRusty Russell .set = param_set_slot_table_size,
36929bbb9e5aSRusty Russell .get = param_get_uint,
36939bbb9e5aSRusty Russell };
36949bbb9e5aSRusty Russell
3695cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \
3696cbf11071STrond Myklebust __param_check(name, p, unsigned int);
3697cbf11071STrond Myklebust
param_set_max_slot_table_size(const char * val,const struct kernel_param * kp)3698d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val,
3699d9ba131dSTrond Myklebust const struct kernel_param *kp)
3700d9ba131dSTrond Myklebust {
3701d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp,
3702d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE,
3703d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT);
3704d9ba131dSTrond Myklebust }
3705d9ba131dSTrond Myklebust
37069c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = {
3707d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size,
3708d9ba131dSTrond Myklebust .get = param_get_uint,
3709d9ba131dSTrond Myklebust };
3710d9ba131dSTrond Myklebust
3711d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \
3712d9ba131dSTrond Myklebust __param_check(name, p, unsigned int);
3713d9ba131dSTrond Myklebust
3714cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries,
3715cbf11071STrond Myklebust slot_table_size, 0644);
3716d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries,
3717d9ba131dSTrond Myklebust max_slot_table_size, 0644);
3718cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries,
3719cbf11071STrond Myklebust slot_table_size, 0644);
3720