1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a246b010SChuck Lever /* 3a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c 4a246b010SChuck Lever * 5a246b010SChuck Lever * Client-side transport implementation for sockets. 6a246b010SChuck Lever * 7113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat 8113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat 9a246b010SChuck Lever * TCP NFS related read + write fixes 10a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 11a246b010SChuck Lever * 12a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff. 13a246b010SChuck Lever * Fix behaviour when socket buffer is full. 14a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no> 1555aa4f58SChuck Lever * 1655aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com> 178f9d5b1aSChuck Lever * 188f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005. 198f9d5b1aSChuck Lever * <gilles.quillard@bull.net> 20a246b010SChuck Lever */ 21a246b010SChuck Lever 22a246b010SChuck Lever #include <linux/types.h> 23176e21eeSChuck Lever #include <linux/string.h> 24a246b010SChuck Lever #include <linux/slab.h> 25bc25571eS\"Talpey, Thomas\ #include <linux/module.h> 26a246b010SChuck Lever #include <linux/capability.h> 27a246b010SChuck Lever #include <linux/pagemap.h> 28a246b010SChuck Lever #include <linux/errno.h> 29a246b010SChuck Lever #include <linux/socket.h> 30a246b010SChuck Lever #include <linux/in.h> 31a246b010SChuck Lever #include <linux/net.h> 32a246b010SChuck Lever #include <linux/mm.h> 33176e21eeSChuck Lever #include <linux/un.h> 34a246b010SChuck Lever #include <linux/udp.h> 35a246b010SChuck Lever #include <linux/tcp.h> 36a246b010SChuck Lever #include <linux/sunrpc/clnt.h> 375976687aSJeff Layton #include <linux/sunrpc/addr.h> 3802107148SChuck Lever #include <linux/sunrpc/sched.h> 394cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h> 4049c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h> 41a246b010SChuck Lever #include <linux/file.h> 429e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL 4344b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h> 4444b98efdSRicardo Labiaga #endif 45a246b010SChuck Lever 46a246b010SChuck Lever #include <net/sock.h> 47a246b010SChuck Lever #include <net/checksum.h> 48a246b010SChuck Lever #include <net/udp.h> 49a246b010SChuck Lever #include <net/tcp.h> 50277e4ab7STrond Myklebust #include <linux/bvec.h> 516a829eb8STrond Myklebust #include <linux/highmem.h> 52277e4ab7STrond Myklebust #include <linux/uio.h> 53a1231fdaSTrond Myklebust #include <linux/sched/mm.h> 54a246b010SChuck Lever 5540b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h> 5640b5ea0cSTrond Myklebust 579e55eef4SChuck Lever #include "socklib.h" 584cfc7e60SRahul Iyer #include "sunrpc.h" 59176e21eeSChuck Lever 60176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt); 617196dbb0STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 627196dbb0STrond Myklebust struct socket *sock); 63176e21eeSChuck Lever 649903cd1cSChuck Lever /* 65c556b754SChuck Lever * xprtsock tunables 66c556b754SChuck Lever */ 6709acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE; 6809acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE; 6909acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE; 70c556b754SChuck Lever 7109acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT; 7209acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT; 73c556b754SChuck Lever 747d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ) 7525fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO; 767d1e8255STrond Myklebust 77c556b754SChuck Lever /* 78fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by 79fbf76683SChuck Lever * calling register_sysctl_table() again. The files in that 80fbf76683SChuck Lever * directory become the union of all files registered there. 81fbf76683SChuck Lever * 82fbf76683SChuck Lever * We simply need to make sure that we don't collide with 83fbf76683SChuck Lever * someone else's file names! 84fbf76683SChuck Lever */ 85fbf76683SChuck Lever 86fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE; 87fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE; 88d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT; 89fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT; 90fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT; 91fbf76683SChuck Lever 92fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header; 93fbf76683SChuck Lever 94fbf76683SChuck Lever /* 95fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP 96fbf76683SChuck Lever * socket buffers for existing UDP transports 97fbf76683SChuck Lever */ 98fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = { 99fbf76683SChuck Lever { 100fbf76683SChuck Lever .procname = "udp_slot_table_entries", 101fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries, 102fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 103fbf76683SChuck Lever .mode = 0644, 1046d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 105fbf76683SChuck Lever .extra1 = &min_slot_table_size, 106fbf76683SChuck Lever .extra2 = &max_slot_table_size 107fbf76683SChuck Lever }, 108fbf76683SChuck Lever { 109fbf76683SChuck Lever .procname = "tcp_slot_table_entries", 110fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries, 111fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 112fbf76683SChuck Lever .mode = 0644, 1136d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 114fbf76683SChuck Lever .extra1 = &min_slot_table_size, 115fbf76683SChuck Lever .extra2 = &max_slot_table_size 116fbf76683SChuck Lever }, 117fbf76683SChuck Lever { 118d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries", 119d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries, 120d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int), 121d9ba131dSTrond Myklebust .mode = 0644, 122d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax, 123d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size, 124d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit 125d9ba131dSTrond Myklebust }, 126d9ba131dSTrond Myklebust { 127fbf76683SChuck Lever .procname = "min_resvport", 128fbf76683SChuck Lever .data = &xprt_min_resvport, 129fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 130fbf76683SChuck Lever .mode = 0644, 1316d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 132fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 133826799e6SJ. Bruce Fields .extra2 = &xprt_max_resvport_limit 134fbf76683SChuck Lever }, 135fbf76683SChuck Lever { 136fbf76683SChuck Lever .procname = "max_resvport", 137fbf76683SChuck Lever .data = &xprt_max_resvport, 138fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 139fbf76683SChuck Lever .mode = 0644, 1406d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 141826799e6SJ. Bruce Fields .extra1 = &xprt_min_resvport_limit, 142fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 143fbf76683SChuck Lever }, 144fbf76683SChuck Lever { 14525fe6142STrond Myklebust .procname = "tcp_fin_timeout", 14625fe6142STrond Myklebust .data = &xs_tcp_fin_timeout, 14725fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout), 14825fe6142STrond Myklebust .mode = 0644, 1496d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies, 15025fe6142STrond Myklebust }, 151f8572d8fSEric W. Biederman { }, 152fbf76683SChuck Lever }; 153fbf76683SChuck Lever 154fe2c6338SJoe Perches static struct ctl_table sunrpc_table[] = { 155fbf76683SChuck Lever { 156fbf76683SChuck Lever .procname = "sunrpc", 157fbf76683SChuck Lever .mode = 0555, 158fbf76683SChuck Lever .child = xs_tunables_table 159fbf76683SChuck Lever }, 160f8572d8fSEric W. Biederman { }, 161fbf76683SChuck Lever }; 162fbf76683SChuck Lever 163fbf76683SChuck Lever /* 16403bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper. 16503bf4b70SChuck Lever */ 16603bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ) 16703bf4b70SChuck Lever 16803bf4b70SChuck Lever /* 16903bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some 17003bf4b70SChuck Lever * kind of resource problem on the local host. 17103bf4b70SChuck Lever */ 17203bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ) 17303bf4b70SChuck Lever 17403bf4b70SChuck Lever /* 17503bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting 17603bf4b70SChuck Lever * to reconnect to a server that just dropped our connection. 17703bf4b70SChuck Lever * 17803bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP 17903bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP 18003bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and 18103bf4b70SChuck Lever * increase over time if the server is down or not responding. 18203bf4b70SChuck Lever */ 18303bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ) 18403bf4b70SChuck Lever 18503bf4b70SChuck Lever /* 18603bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle 18703bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent 18803bf4b70SChuck Lever * holding port numbers when there is no RPC traffic. 18903bf4b70SChuck Lever */ 19003bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ) 19103bf4b70SChuck Lever 192f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 193a246b010SChuck Lever # undef RPC_DEBUG_DATA 1949903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS 195a246b010SChuck Lever #endif 196a246b010SChuck Lever 197a246b010SChuck Lever #ifdef RPC_DEBUG_DATA 1989903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count) 199a246b010SChuck Lever { 200a246b010SChuck Lever u8 *buf = (u8 *) packet; 201a246b010SChuck Lever int j; 202a246b010SChuck Lever 203a246b010SChuck Lever dprintk("RPC: %s\n", msg); 204a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) { 205a246b010SChuck Lever if (!(j & 31)) { 206a246b010SChuck Lever if (j) 207a246b010SChuck Lever dprintk("\n"); 208a246b010SChuck Lever dprintk("0x%04x ", j); 209a246b010SChuck Lever } 210a246b010SChuck Lever dprintk("%02x%02x%02x%02x ", 211a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]); 212a246b010SChuck Lever } 213a246b010SChuck Lever dprintk("\n"); 214a246b010SChuck Lever } 215a246b010SChuck Lever #else 2169903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) 217a246b010SChuck Lever { 218a246b010SChuck Lever /* NOP */ 219a246b010SChuck Lever } 220a246b010SChuck Lever #endif 221a246b010SChuck Lever 2222118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) 2232118071dSTrond Myklebust { 2242118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data; 2252118071dSTrond Myklebust } 2262118071dSTrond Myklebust 22795392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) 228edb267a6SChuck Lever { 22995392c59SChuck Lever return (struct sockaddr *) &xprt->addr; 23095392c59SChuck Lever } 23195392c59SChuck Lever 232176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) 233176e21eeSChuck Lever { 234176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr; 235176e21eeSChuck Lever } 236176e21eeSChuck Lever 23795392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) 23895392c59SChuck Lever { 23995392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr; 24095392c59SChuck Lever } 24195392c59SChuck Lever 24295392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) 24395392c59SChuck Lever { 24495392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr; 24595392c59SChuck Lever } 24695392c59SChuck Lever 247c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) 248c877b849SChuck Lever { 249c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2509dc3b095SChuck Lever struct sockaddr_in6 *sin6; 2519dc3b095SChuck Lever struct sockaddr_in *sin; 252176e21eeSChuck Lever struct sockaddr_un *sun; 253c877b849SChuck Lever char buf[128]; 254c877b849SChuck Lever 2559dc3b095SChuck Lever switch (sap->sa_family) { 256176e21eeSChuck Lever case AF_LOCAL: 257176e21eeSChuck Lever sun = xs_addr_un(xprt); 258176e21eeSChuck Lever strlcpy(buf, sun->sun_path, sizeof(buf)); 259176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 260176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 261176e21eeSChuck Lever break; 2629dc3b095SChuck Lever case AF_INET: 263176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 264176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 265176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2669dc3b095SChuck Lever sin = xs_addr_in(xprt); 267fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); 2689dc3b095SChuck Lever break; 2699dc3b095SChuck Lever case AF_INET6: 270176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 271176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 272176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2739dc3b095SChuck Lever sin6 = xs_addr_in6(xprt); 274fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); 2759dc3b095SChuck Lever break; 2769dc3b095SChuck Lever default: 2779dc3b095SChuck Lever BUG(); 2789dc3b095SChuck Lever } 279176e21eeSChuck Lever 2809dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); 2819dc3b095SChuck Lever } 2829dc3b095SChuck Lever 2839dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt) 2849dc3b095SChuck Lever { 2859dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2869dc3b095SChuck Lever char buf[128]; 2879dc3b095SChuck Lever 28881160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); 289c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); 290c877b849SChuck Lever 29181160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); 292c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); 293c877b849SChuck Lever } 294c877b849SChuck Lever 2959dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt, 296b454ae90SChuck Lever const char *protocol, 297b454ae90SChuck Lever const char *netid) 298edb267a6SChuck Lever { 299b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; 300b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid; 301c877b849SChuck Lever xs_format_common_peer_addresses(xprt); 3029dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 303edb267a6SChuck Lever } 304edb267a6SChuck Lever 3059dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt) 3064b6473fbSChuck Lever { 3079dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); 3089dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]); 3094b6473fbSChuck Lever 3109dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 311edb267a6SChuck Lever } 312edb267a6SChuck Lever 313edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt) 314edb267a6SChuck Lever { 31533e01dc7SChuck Lever unsigned int i; 31633e01dc7SChuck Lever 31733e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++) 31833e01dc7SChuck Lever switch (i) { 31933e01dc7SChuck Lever case RPC_DISPLAY_PROTO: 32033e01dc7SChuck Lever case RPC_DISPLAY_NETID: 32133e01dc7SChuck Lever continue; 32233e01dc7SChuck Lever default: 32333e01dc7SChuck Lever kfree(xprt->address_strings[i]); 32433e01dc7SChuck Lever } 325edb267a6SChuck Lever } 326edb267a6SChuck Lever 327277e4ab7STrond Myklebust static size_t 328277e4ab7STrond Myklebust xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) 329277e4ab7STrond Myklebust { 330277e4ab7STrond Myklebust size_t i,n; 331277e4ab7STrond Myklebust 33216e5e90fSTrond Myklebust if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES)) 333277e4ab7STrond Myklebust return want; 334277e4ab7STrond Myklebust n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT; 335277e4ab7STrond Myklebust for (i = 0; i < n; i++) { 336277e4ab7STrond Myklebust if (buf->pages[i]) 337277e4ab7STrond Myklebust continue; 338277e4ab7STrond Myklebust buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp); 339277e4ab7STrond Myklebust if (!buf->pages[i]) { 34016e5e90fSTrond Myklebust i *= PAGE_SIZE; 34116e5e90fSTrond Myklebust return i > buf->page_base ? i - buf->page_base : 0; 342277e4ab7STrond Myklebust } 343277e4ab7STrond Myklebust } 344277e4ab7STrond Myklebust return want; 345277e4ab7STrond Myklebust } 346277e4ab7STrond Myklebust 347277e4ab7STrond Myklebust static ssize_t 348277e4ab7STrond Myklebust xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) 349277e4ab7STrond Myklebust { 350277e4ab7STrond Myklebust ssize_t ret; 351277e4ab7STrond Myklebust if (seek != 0) 352277e4ab7STrond Myklebust iov_iter_advance(&msg->msg_iter, seek); 353277e4ab7STrond Myklebust ret = sock_recvmsg(sock, msg, flags); 354277e4ab7STrond Myklebust return ret > 0 ? ret + seek : ret; 355277e4ab7STrond Myklebust } 356277e4ab7STrond Myklebust 357277e4ab7STrond Myklebust static ssize_t 358277e4ab7STrond Myklebust xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags, 359277e4ab7STrond Myklebust struct kvec *kvec, size_t count, size_t seek) 360277e4ab7STrond Myklebust { 3610e9b4a82SAl Viro iov_iter_kvec(&msg->msg_iter, READ, kvec, 1, count); 362277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 363277e4ab7STrond Myklebust } 364277e4ab7STrond Myklebust 365277e4ab7STrond Myklebust static ssize_t 366277e4ab7STrond Myklebust xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags, 367277e4ab7STrond Myklebust struct bio_vec *bvec, unsigned long nr, size_t count, 368277e4ab7STrond Myklebust size_t seek) 369277e4ab7STrond Myklebust { 3700e9b4a82SAl Viro iov_iter_bvec(&msg->msg_iter, READ, bvec, nr, count); 371277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 372277e4ab7STrond Myklebust } 373277e4ab7STrond Myklebust 374277e4ab7STrond Myklebust static ssize_t 375277e4ab7STrond Myklebust xs_read_discard(struct socket *sock, struct msghdr *msg, int flags, 376277e4ab7STrond Myklebust size_t count) 377277e4ab7STrond Myklebust { 378b76a5afdSTrond Myklebust iov_iter_discard(&msg->msg_iter, READ, count); 379b76a5afdSTrond Myklebust return sock_recvmsg(sock, msg, flags); 380277e4ab7STrond Myklebust } 381277e4ab7STrond Myklebust 3826a829eb8STrond Myklebust #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 3836a829eb8STrond Myklebust static void 3846a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek) 3856a829eb8STrond Myklebust { 3866a829eb8STrond Myklebust struct bvec_iter bi = { 3876a829eb8STrond Myklebust .bi_size = count, 3886a829eb8STrond Myklebust }; 3896a829eb8STrond Myklebust struct bio_vec bv; 3906a829eb8STrond Myklebust 3916a829eb8STrond Myklebust bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); 3926a829eb8STrond Myklebust for_each_bvec(bv, bvec, bi, bi) 3936a829eb8STrond Myklebust flush_dcache_page(bv.bv_page); 3946a829eb8STrond Myklebust } 3956a829eb8STrond Myklebust #else 3966a829eb8STrond Myklebust static inline void 3976a829eb8STrond Myklebust xs_flush_bvec(const struct bio_vec *bvec, size_t count, size_t seek) 3986a829eb8STrond Myklebust { 3996a829eb8STrond Myklebust } 4006a829eb8STrond Myklebust #endif 4016a829eb8STrond Myklebust 402277e4ab7STrond Myklebust static ssize_t 403277e4ab7STrond Myklebust xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags, 404277e4ab7STrond Myklebust struct xdr_buf *buf, size_t count, size_t seek, size_t *read) 405277e4ab7STrond Myklebust { 406277e4ab7STrond Myklebust size_t want, seek_init = seek, offset = 0; 407277e4ab7STrond Myklebust ssize_t ret; 408277e4ab7STrond Myklebust 409277e4ab7STrond Myklebust want = min_t(size_t, count, buf->head[0].iov_len); 410e92053a5STrond Myklebust if (seek < want) { 411277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek); 412277e4ab7STrond Myklebust if (ret <= 0) 413277e4ab7STrond Myklebust goto sock_err; 414277e4ab7STrond Myklebust offset += ret; 415277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 416277e4ab7STrond Myklebust goto out; 417277e4ab7STrond Myklebust if (ret != want) 418c4433055STrond Myklebust goto out; 419277e4ab7STrond Myklebust seek = 0; 420277e4ab7STrond Myklebust } else { 421e92053a5STrond Myklebust seek -= want; 422e92053a5STrond Myklebust offset += want; 423277e4ab7STrond Myklebust } 42416e5e90fSTrond Myklebust 425277e4ab7STrond Myklebust want = xs_alloc_sparse_pages(buf, 426277e4ab7STrond Myklebust min_t(size_t, count - offset, buf->page_len), 42712a3ad61STrond Myklebust GFP_KERNEL); 42816e5e90fSTrond Myklebust if (seek < want) { 429277e4ab7STrond Myklebust ret = xs_read_bvec(sock, msg, flags, buf->bvec, 430277e4ab7STrond Myklebust xdr_buf_pagecount(buf), 431277e4ab7STrond Myklebust want + buf->page_base, 432277e4ab7STrond Myklebust seek + buf->page_base); 433277e4ab7STrond Myklebust if (ret <= 0) 434277e4ab7STrond Myklebust goto sock_err; 4356a829eb8STrond Myklebust xs_flush_bvec(buf->bvec, ret, seek + buf->page_base); 436ac9645c8SDan Aloni ret -= buf->page_base; 437ac9645c8SDan Aloni offset += ret; 438277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 439277e4ab7STrond Myklebust goto out; 440277e4ab7STrond Myklebust if (ret != want) 441c4433055STrond Myklebust goto out; 442277e4ab7STrond Myklebust seek = 0; 443277e4ab7STrond Myklebust } else { 44416e5e90fSTrond Myklebust seek -= want; 44516e5e90fSTrond Myklebust offset += want; 446277e4ab7STrond Myklebust } 44716e5e90fSTrond Myklebust 448277e4ab7STrond Myklebust want = min_t(size_t, count - offset, buf->tail[0].iov_len); 449e92053a5STrond Myklebust if (seek < want) { 450277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek); 451277e4ab7STrond Myklebust if (ret <= 0) 452277e4ab7STrond Myklebust goto sock_err; 453277e4ab7STrond Myklebust offset += ret; 454277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 455277e4ab7STrond Myklebust goto out; 456277e4ab7STrond Myklebust if (ret != want) 457c4433055STrond Myklebust goto out; 4589734ad57STrond Myklebust } else if (offset < seek_init) 459e92053a5STrond Myklebust offset = seek_init; 460277e4ab7STrond Myklebust ret = -EMSGSIZE; 461277e4ab7STrond Myklebust out: 462277e4ab7STrond Myklebust *read = offset - seek_init; 463277e4ab7STrond Myklebust return ret; 464277e4ab7STrond Myklebust sock_err: 465277e4ab7STrond Myklebust offset += seek; 466277e4ab7STrond Myklebust goto out; 467277e4ab7STrond Myklebust } 468277e4ab7STrond Myklebust 469277e4ab7STrond Myklebust static void 470277e4ab7STrond Myklebust xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf) 471277e4ab7STrond Myklebust { 472277e4ab7STrond Myklebust if (!transport->recv.copied) { 473277e4ab7STrond Myklebust if (buf->head[0].iov_len >= transport->recv.offset) 474277e4ab7STrond Myklebust memcpy(buf->head[0].iov_base, 475277e4ab7STrond Myklebust &transport->recv.xid, 476277e4ab7STrond Myklebust transport->recv.offset); 477277e4ab7STrond Myklebust transport->recv.copied = transport->recv.offset; 478277e4ab7STrond Myklebust } 479277e4ab7STrond Myklebust } 480277e4ab7STrond Myklebust 481277e4ab7STrond Myklebust static bool 482277e4ab7STrond Myklebust xs_read_stream_request_done(struct sock_xprt *transport) 483277e4ab7STrond Myklebust { 484277e4ab7STrond Myklebust return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT); 485277e4ab7STrond Myklebust } 486277e4ab7STrond Myklebust 487e92053a5STrond Myklebust static void 488e92053a5STrond Myklebust xs_read_stream_check_eor(struct sock_xprt *transport, 489e92053a5STrond Myklebust struct msghdr *msg) 490e92053a5STrond Myklebust { 491e92053a5STrond Myklebust if (xs_read_stream_request_done(transport)) 492e92053a5STrond Myklebust msg->msg_flags |= MSG_EOR; 493e92053a5STrond Myklebust } 494e92053a5STrond Myklebust 495277e4ab7STrond Myklebust static ssize_t 496277e4ab7STrond Myklebust xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg, 497277e4ab7STrond Myklebust int flags, struct rpc_rqst *req) 498277e4ab7STrond Myklebust { 499277e4ab7STrond Myklebust struct xdr_buf *buf = &req->rq_private_buf; 5003f649ab7SKees Cook size_t want, read; 5013f649ab7SKees Cook ssize_t ret; 502277e4ab7STrond Myklebust 503277e4ab7STrond Myklebust xs_read_header(transport, buf); 504277e4ab7STrond Myklebust 505277e4ab7STrond Myklebust want = transport->recv.len - transport->recv.offset; 506e92053a5STrond Myklebust if (want != 0) { 507277e4ab7STrond Myklebust ret = xs_read_xdr_buf(transport->sock, msg, flags, buf, 508e92053a5STrond Myklebust transport->recv.copied + want, 509e92053a5STrond Myklebust transport->recv.copied, 510277e4ab7STrond Myklebust &read); 511277e4ab7STrond Myklebust transport->recv.offset += read; 512277e4ab7STrond Myklebust transport->recv.copied += read; 513277e4ab7STrond Myklebust } 514277e4ab7STrond Myklebust 515727fcc64STrond Myklebust if (transport->recv.offset == transport->recv.len) 516727fcc64STrond Myklebust xs_read_stream_check_eor(transport, msg); 517727fcc64STrond Myklebust 518e92053a5STrond Myklebust if (want == 0) 519e92053a5STrond Myklebust return 0; 520e92053a5STrond Myklebust 521277e4ab7STrond Myklebust switch (ret) { 522c4433055STrond Myklebust default: 523c4433055STrond Myklebust break; 52426781eabSTrond Myklebust case -EFAULT: 525277e4ab7STrond Myklebust case -EMSGSIZE: 52626781eabSTrond Myklebust msg->msg_flags |= MSG_TRUNC; 527c4433055STrond Myklebust return read; 528277e4ab7STrond Myklebust case 0: 529277e4ab7STrond Myklebust return -ESHUTDOWN; 530277e4ab7STrond Myklebust } 531c4433055STrond Myklebust return ret < 0 ? ret : read; 532277e4ab7STrond Myklebust } 533277e4ab7STrond Myklebust 534277e4ab7STrond Myklebust static size_t 535277e4ab7STrond Myklebust xs_read_stream_headersize(bool isfrag) 536277e4ab7STrond Myklebust { 537277e4ab7STrond Myklebust if (isfrag) 538277e4ab7STrond Myklebust return sizeof(__be32); 539277e4ab7STrond Myklebust return 3 * sizeof(__be32); 540277e4ab7STrond Myklebust } 541277e4ab7STrond Myklebust 542277e4ab7STrond Myklebust static ssize_t 543277e4ab7STrond Myklebust xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg, 544277e4ab7STrond Myklebust int flags, size_t want, size_t seek) 545277e4ab7STrond Myklebust { 546277e4ab7STrond Myklebust struct kvec kvec = { 547277e4ab7STrond Myklebust .iov_base = &transport->recv.fraghdr, 548277e4ab7STrond Myklebust .iov_len = want, 549277e4ab7STrond Myklebust }; 550277e4ab7STrond Myklebust return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek); 551277e4ab7STrond Myklebust } 552277e4ab7STrond Myklebust 553277e4ab7STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 554277e4ab7STrond Myklebust static ssize_t 555277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 556277e4ab7STrond Myklebust { 557277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 558277e4ab7STrond Myklebust struct rpc_rqst *req; 559277e4ab7STrond Myklebust ssize_t ret; 560277e4ab7STrond Myklebust 561*98b5cee3SBenjamin Coddington /* Is this transport associated with the backchannel? */ 562*98b5cee3SBenjamin Coddington if (!xprt->bc_serv) 563*98b5cee3SBenjamin Coddington return -ESHUTDOWN; 564*98b5cee3SBenjamin Coddington 565277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 566277e4ab7STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->recv.xid); 567277e4ab7STrond Myklebust if (!req) { 568277e4ab7STrond Myklebust printk(KERN_WARNING "Callback slot table overflowed\n"); 569277e4ab7STrond Myklebust return -ESHUTDOWN; 570277e4ab7STrond Myklebust } 57145835a63STrond Myklebust if (transport->recv.copied && !req->rq_private_buf.len) 57245835a63STrond Myklebust return -ESHUTDOWN; 573277e4ab7STrond Myklebust 574277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 575277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 576c4433055STrond Myklebust xprt_complete_bc_request(req, transport->recv.copied); 57745835a63STrond Myklebust else 57845835a63STrond Myklebust req->rq_private_buf.len = transport->recv.copied; 579277e4ab7STrond Myklebust 580277e4ab7STrond Myklebust return ret; 581277e4ab7STrond Myklebust } 582277e4ab7STrond Myklebust #else /* CONFIG_SUNRPC_BACKCHANNEL */ 583277e4ab7STrond Myklebust static ssize_t 584277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 585277e4ab7STrond Myklebust { 586277e4ab7STrond Myklebust return -ESHUTDOWN; 587277e4ab7STrond Myklebust } 588277e4ab7STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 589277e4ab7STrond Myklebust 590277e4ab7STrond Myklebust static ssize_t 591277e4ab7STrond Myklebust xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags) 592277e4ab7STrond Myklebust { 593277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 594277e4ab7STrond Myklebust struct rpc_rqst *req; 595277e4ab7STrond Myklebust ssize_t ret = 0; 596277e4ab7STrond Myklebust 597277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 598277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 599277e4ab7STrond Myklebust req = xprt_lookup_rqst(xprt, transport->recv.xid); 60045835a63STrond Myklebust if (!req || (transport->recv.copied && !req->rq_private_buf.len)) { 601277e4ab7STrond Myklebust msg->msg_flags |= MSG_TRUNC; 602277e4ab7STrond Myklebust goto out; 603277e4ab7STrond Myklebust } 604277e4ab7STrond Myklebust xprt_pin_rqst(req); 605277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 606277e4ab7STrond Myklebust 607277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 608277e4ab7STrond Myklebust 609277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 610277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 611c4433055STrond Myklebust xprt_complete_rqst(req->rq_task, transport->recv.copied); 61245835a63STrond Myklebust else 61345835a63STrond Myklebust req->rq_private_buf.len = transport->recv.copied; 614277e4ab7STrond Myklebust xprt_unpin_rqst(req); 615277e4ab7STrond Myklebust out: 616277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 617277e4ab7STrond Myklebust return ret; 618277e4ab7STrond Myklebust } 619277e4ab7STrond Myklebust 620277e4ab7STrond Myklebust static ssize_t 621277e4ab7STrond Myklebust xs_read_stream(struct sock_xprt *transport, int flags) 622277e4ab7STrond Myklebust { 623277e4ab7STrond Myklebust struct msghdr msg = { 0 }; 624277e4ab7STrond Myklebust size_t want, read = 0; 625277e4ab7STrond Myklebust ssize_t ret = 0; 626277e4ab7STrond Myklebust 627277e4ab7STrond Myklebust if (transport->recv.len == 0) { 628277e4ab7STrond Myklebust want = xs_read_stream_headersize(transport->recv.copied != 0); 629277e4ab7STrond Myklebust ret = xs_read_stream_header(transport, &msg, flags, want, 630277e4ab7STrond Myklebust transport->recv.offset); 631277e4ab7STrond Myklebust if (ret <= 0) 632277e4ab7STrond Myklebust goto out_err; 633277e4ab7STrond Myklebust transport->recv.offset = ret; 634c4433055STrond Myklebust if (transport->recv.offset != want) 635c4433055STrond Myklebust return transport->recv.offset; 636277e4ab7STrond Myklebust transport->recv.len = be32_to_cpu(transport->recv.fraghdr) & 637277e4ab7STrond Myklebust RPC_FRAGMENT_SIZE_MASK; 638277e4ab7STrond Myklebust transport->recv.offset -= sizeof(transport->recv.fraghdr); 639277e4ab7STrond Myklebust read = ret; 640277e4ab7STrond Myklebust } 641277e4ab7STrond Myklebust 642277e4ab7STrond Myklebust switch (be32_to_cpu(transport->recv.calldir)) { 643c4433055STrond Myklebust default: 644c4433055STrond Myklebust msg.msg_flags |= MSG_TRUNC; 645c4433055STrond Myklebust break; 646277e4ab7STrond Myklebust case RPC_CALL: 647277e4ab7STrond Myklebust ret = xs_read_stream_call(transport, &msg, flags); 648277e4ab7STrond Myklebust break; 649277e4ab7STrond Myklebust case RPC_REPLY: 650277e4ab7STrond Myklebust ret = xs_read_stream_reply(transport, &msg, flags); 651277e4ab7STrond Myklebust } 652277e4ab7STrond Myklebust if (msg.msg_flags & MSG_TRUNC) { 653277e4ab7STrond Myklebust transport->recv.calldir = cpu_to_be32(-1); 654277e4ab7STrond Myklebust transport->recv.copied = -1; 655277e4ab7STrond Myklebust } 656277e4ab7STrond Myklebust if (ret < 0) 657277e4ab7STrond Myklebust goto out_err; 658277e4ab7STrond Myklebust read += ret; 659277e4ab7STrond Myklebust if (transport->recv.offset < transport->recv.len) { 660c4433055STrond Myklebust if (!(msg.msg_flags & MSG_TRUNC)) 661c4433055STrond Myklebust return read; 662b76a5afdSTrond Myklebust msg.msg_flags = 0; 663277e4ab7STrond Myklebust ret = xs_read_discard(transport->sock, &msg, flags, 664277e4ab7STrond Myklebust transport->recv.len - transport->recv.offset); 665277e4ab7STrond Myklebust if (ret <= 0) 666277e4ab7STrond Myklebust goto out_err; 667277e4ab7STrond Myklebust transport->recv.offset += ret; 668277e4ab7STrond Myklebust read += ret; 669277e4ab7STrond Myklebust if (transport->recv.offset != transport->recv.len) 670c4433055STrond Myklebust return read; 671277e4ab7STrond Myklebust } 672277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) { 673c50b8ee0STrond Myklebust trace_xs_stream_read_request(transport); 674277e4ab7STrond Myklebust transport->recv.copied = 0; 675277e4ab7STrond Myklebust } 676277e4ab7STrond Myklebust transport->recv.offset = 0; 677277e4ab7STrond Myklebust transport->recv.len = 0; 678277e4ab7STrond Myklebust return read; 679277e4ab7STrond Myklebust out_err: 68079462857STrond Myklebust return ret != 0 ? ret : -ESHUTDOWN; 681277e4ab7STrond Myklebust } 682277e4ab7STrond Myklebust 6830ffe86f4STrond Myklebust static __poll_t xs_poll_socket(struct sock_xprt *transport) 6840ffe86f4STrond Myklebust { 685a73881c9STrond Myklebust return transport->sock->ops->poll(transport->file, transport->sock, 686a73881c9STrond Myklebust NULL); 6870ffe86f4STrond Myklebust } 6880ffe86f4STrond Myklebust 6890ffe86f4STrond Myklebust static bool xs_poll_socket_readable(struct sock_xprt *transport) 6900ffe86f4STrond Myklebust { 6910ffe86f4STrond Myklebust __poll_t events = xs_poll_socket(transport); 6920ffe86f4STrond Myklebust 6930ffe86f4STrond Myklebust return (events & (EPOLLIN | EPOLLRDNORM)) && !(events & EPOLLRDHUP); 6940ffe86f4STrond Myklebust } 6950ffe86f4STrond Myklebust 6960ffe86f4STrond Myklebust static void xs_poll_check_readable(struct sock_xprt *transport) 6970ffe86f4STrond Myklebust { 6980ffe86f4STrond Myklebust 6990ffe86f4STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 7000ffe86f4STrond Myklebust if (!xs_poll_socket_readable(transport)) 7010ffe86f4STrond Myklebust return; 7020ffe86f4STrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 7030ffe86f4STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 7040ffe86f4STrond Myklebust } 7050ffe86f4STrond Myklebust 706c50b8ee0STrond Myklebust static void xs_stream_data_receive(struct sock_xprt *transport) 707c50b8ee0STrond Myklebust { 708c50b8ee0STrond Myklebust size_t read = 0; 709c50b8ee0STrond Myklebust ssize_t ret = 0; 710c50b8ee0STrond Myklebust 711c50b8ee0STrond Myklebust mutex_lock(&transport->recv_mutex); 712c50b8ee0STrond Myklebust if (transport->sock == NULL) 713c50b8ee0STrond Myklebust goto out; 714c50b8ee0STrond Myklebust for (;;) { 715c50b8ee0STrond Myklebust ret = xs_read_stream(transport, MSG_DONTWAIT); 716c4433055STrond Myklebust if (ret < 0) 717c50b8ee0STrond Myklebust break; 718c50b8ee0STrond Myklebust read += ret; 719c50b8ee0STrond Myklebust cond_resched(); 720c50b8ee0STrond Myklebust } 7215f52a9d4STrond Myklebust if (ret == -ESHUTDOWN) 7225f52a9d4STrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR); 7235f52a9d4STrond Myklebust else 7240ffe86f4STrond Myklebust xs_poll_check_readable(transport); 725c50b8ee0STrond Myklebust out: 726c50b8ee0STrond Myklebust mutex_unlock(&transport->recv_mutex); 727c50b8ee0STrond Myklebust trace_xs_stream_read_data(&transport->xprt, ret, read); 728c50b8ee0STrond Myklebust } 729c50b8ee0STrond Myklebust 730c50b8ee0STrond Myklebust static void xs_stream_data_receive_workfn(struct work_struct *work) 731c50b8ee0STrond Myklebust { 732c50b8ee0STrond Myklebust struct sock_xprt *transport = 733c50b8ee0STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 734a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 735a1231fdaSTrond Myklebust 736c50b8ee0STrond Myklebust xs_stream_data_receive(transport); 737a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 738c50b8ee0STrond Myklebust } 739c50b8ee0STrond Myklebust 740550aebfeSTrond Myklebust static void 741550aebfeSTrond Myklebust xs_stream_reset_connect(struct sock_xprt *transport) 742550aebfeSTrond Myklebust { 743550aebfeSTrond Myklebust transport->recv.offset = 0; 744550aebfeSTrond Myklebust transport->recv.len = 0; 745550aebfeSTrond Myklebust transport->recv.copied = 0; 746550aebfeSTrond Myklebust transport->xmit.offset = 0; 747ae053551STrond Myklebust } 748ae053551STrond Myklebust 749ae053551STrond Myklebust static void 750ae053551STrond Myklebust xs_stream_start_connect(struct sock_xprt *transport) 751ae053551STrond Myklebust { 752550aebfeSTrond Myklebust transport->xprt.stat.connect_count++; 753550aebfeSTrond Myklebust transport->xprt.stat.connect_start = jiffies; 754550aebfeSTrond Myklebust } 755550aebfeSTrond Myklebust 756b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 757b4b5cc85SChuck Lever 7589903cd1cSChuck Lever /** 759c544577dSTrond Myklebust * xs_nospace - handle transmit was incomplete 76050f484e2STrond Myklebust * @req: pointer to RPC request 7619903cd1cSChuck Lever * 762a246b010SChuck Lever */ 763c544577dSTrond Myklebust static int xs_nospace(struct rpc_rqst *req) 764a246b010SChuck Lever { 765262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 766ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 76706ea0bfeSTrond Myklebust struct sock *sk = transport->inet; 76824ca9a84STrond Myklebust int ret = -EAGAIN; 769a246b010SChuck Lever 770015747d2SChuck Lever trace_rpc_socket_nospace(req, transport); 771a246b010SChuck Lever 772262965f5SChuck Lever /* Protect against races with write_space */ 773b5e92419STrond Myklebust spin_lock(&xprt->transport_lock); 774a246b010SChuck Lever 775262965f5SChuck Lever /* Don't race with disconnect */ 776b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) { 77713331a55STrond Myklebust /* wait for more buffer space */ 77806ea0bfeSTrond Myklebust sk->sk_write_pending++; 779c544577dSTrond Myklebust xprt_wait_for_buffer_space(xprt); 78013331a55STrond Myklebust } else 7815e3771ceSTrond Myklebust ret = -ENOTCONN; 782a246b010SChuck Lever 783b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock); 78406ea0bfeSTrond Myklebust 78506ea0bfeSTrond Myklebust /* Race breaker in case memory is freed before above code is called */ 786d48f9ce7SDavid Vrabel if (ret == -EAGAIN) { 787d48f9ce7SDavid Vrabel struct socket_wq *wq; 788d48f9ce7SDavid Vrabel 789d48f9ce7SDavid Vrabel rcu_read_lock(); 790d48f9ce7SDavid Vrabel wq = rcu_dereference(sk->sk_wq); 791d48f9ce7SDavid Vrabel set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags); 792d48f9ce7SDavid Vrabel rcu_read_unlock(); 793d48f9ce7SDavid Vrabel 79406ea0bfeSTrond Myklebust sk->sk_write_space(sk); 795d48f9ce7SDavid Vrabel } 7965e3771ceSTrond Myklebust return ret; 797a246b010SChuck Lever } 798a246b010SChuck Lever 799277e4ab7STrond Myklebust static void 800277e4ab7STrond Myklebust xs_stream_prepare_request(struct rpc_rqst *req) 801277e4ab7STrond Myklebust { 80275369089STrond Myklebust xdr_free_bvec(&req->rq_rcv_buf); 80312a3ad61STrond Myklebust req->rq_task->tk_status = xdr_alloc_bvec(&req->rq_rcv_buf, GFP_KERNEL); 804277e4ab7STrond Myklebust } 805277e4ab7STrond Myklebust 80661677eeeSChuck Lever /* 8074cd34e7cSTrond Myklebust * Determine if the previous message in the stream was aborted before it 8084cd34e7cSTrond Myklebust * could complete transmission. 8094cd34e7cSTrond Myklebust */ 8104cd34e7cSTrond Myklebust static bool 8114cd34e7cSTrond Myklebust xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req) 8124cd34e7cSTrond Myklebust { 8134cd34e7cSTrond Myklebust return transport->xmit.offset != 0 && req->rq_bytes_sent == 0; 8144cd34e7cSTrond Myklebust } 8154cd34e7cSTrond Myklebust 8164cd34e7cSTrond Myklebust /* 81706b5fc3aSTrond Myklebust * Return the stream record marker field for a record of length < 2^31-1 81861677eeeSChuck Lever */ 81906b5fc3aSTrond Myklebust static rpc_fraghdr 82006b5fc3aSTrond Myklebust xs_stream_record_marker(struct xdr_buf *xdr) 82161677eeeSChuck Lever { 82206b5fc3aSTrond Myklebust if (!xdr->len) 82306b5fc3aSTrond Myklebust return 0; 82406b5fc3aSTrond Myklebust return cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | (u32)xdr->len); 82561677eeeSChuck Lever } 82661677eeeSChuck Lever 8279903cd1cSChuck Lever /** 828176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket 82950f484e2STrond Myklebust * @req: pointer to RPC request 830176e21eeSChuck Lever * 831176e21eeSChuck Lever * Return values: 832176e21eeSChuck Lever * 0: The request has been sent 833176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to 834176e21eeSChuck Lever * complete the request 835176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 83612b20ce3SBhaskar Chowdhury * other: Some other error occurred, the request was not sent 837176e21eeSChuck Lever */ 838adfa7144STrond Myklebust static int xs_local_send_request(struct rpc_rqst *req) 839176e21eeSChuck Lever { 840176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 841176e21eeSChuck Lever struct sock_xprt *transport = 842176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt); 843176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 8447e3d3620STrond Myklebust rpc_fraghdr rm = xs_stream_record_marker(xdr); 8457e3d3620STrond Myklebust unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen; 8469e55eef4SChuck Lever struct msghdr msg = { 8479e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS, 8489e55eef4SChuck Lever }; 8493f649ab7SKees Cook unsigned int sent; 850176e21eeSChuck Lever int status; 851176e21eeSChuck Lever 8524cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 8534cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 8544cd34e7cSTrond Myklebust xs_close(xprt); 8554cd34e7cSTrond Myklebust return -ENOTCONN; 8564cd34e7cSTrond Myklebust } 8574cd34e7cSTrond Myklebust 858176e21eeSChuck Lever xs_pktdump("packet data:", 859176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len); 860176e21eeSChuck Lever 86178215759SChuck Lever req->rq_xtime = ktime_get(); 8629e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 8637e3d3620STrond Myklebust transport->xmit.offset, rm, &sent); 864176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n", 8656c7a64e5STrond Myklebust __func__, xdr->len - transport->xmit.offset, status); 866743c69e7SNeilBrown 867743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 868743c69e7SNeilBrown status = -ENOBUFS; 869743c69e7SNeilBrown 870f279cd00SJason Baron if (likely(sent > 0) || status == 0) { 8716c7a64e5STrond Myklebust transport->xmit.offset += sent; 8726c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 8737e3d3620STrond Myklebust if (likely(req->rq_bytes_sent >= msglen)) { 8746c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 8756c7a64e5STrond Myklebust transport->xmit.offset = 0; 876176e21eeSChuck Lever return 0; 877176e21eeSChuck Lever } 878176e21eeSChuck Lever status = -EAGAIN; 879176e21eeSChuck Lever } 880176e21eeSChuck Lever 881176e21eeSChuck Lever switch (status) { 8823601c4a9STrond Myklebust case -ENOBUFS: 883b5872f0cSTrond Myklebust break; 884176e21eeSChuck Lever case -EAGAIN: 885c544577dSTrond Myklebust status = xs_nospace(req); 886176e21eeSChuck Lever break; 887176e21eeSChuck Lever default: 888176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 889176e21eeSChuck Lever -status); 890df561f66SGustavo A. R. Silva fallthrough; 891176e21eeSChuck Lever case -EPIPE: 892176e21eeSChuck Lever xs_close(xprt); 893176e21eeSChuck Lever status = -ENOTCONN; 894176e21eeSChuck Lever } 895176e21eeSChuck Lever 896176e21eeSChuck Lever return status; 897176e21eeSChuck Lever } 898176e21eeSChuck Lever 899176e21eeSChuck Lever /** 900262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket 90150f484e2STrond Myklebust * @req: pointer to RPC request 9029903cd1cSChuck Lever * 9039903cd1cSChuck Lever * Return values: 9049903cd1cSChuck Lever * 0: The request has been sent 9059903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to 9069903cd1cSChuck Lever * complete the request 907262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 90825985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 9099903cd1cSChuck Lever */ 910adfa7144STrond Myklebust static int xs_udp_send_request(struct rpc_rqst *req) 911a246b010SChuck Lever { 912a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 913ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 914262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 9159e55eef4SChuck Lever struct msghdr msg = { 9169e55eef4SChuck Lever .msg_name = xs_addr(xprt), 9179e55eef4SChuck Lever .msg_namelen = xprt->addrlen, 9189e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS, 9199e55eef4SChuck Lever }; 9203f649ab7SKees Cook unsigned int sent; 921262965f5SChuck Lever int status; 922262965f5SChuck Lever 923262965f5SChuck Lever xs_pktdump("packet data:", 924262965f5SChuck Lever req->rq_svec->iov_base, 925262965f5SChuck Lever req->rq_svec->iov_len); 926262965f5SChuck Lever 92701d37c42STrond Myklebust if (!xprt_bound(xprt)) 92801d37c42STrond Myklebust return -ENOTCONN; 92975891f50STrond Myklebust 93075891f50STrond Myklebust if (!xprt_request_get_cong(xprt, req)) 93175891f50STrond Myklebust return -EBADSLT; 93275891f50STrond Myklebust 93378215759SChuck Lever req->rq_xtime = ktime_get(); 9349e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, 0, &sent); 935262965f5SChuck Lever 936262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n", 9376c7a64e5STrond Myklebust xdr->len, status); 938262965f5SChuck Lever 9393dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */ 9403dedbb5cSJason Baron if (status == -EPERM) 9413dedbb5cSJason Baron goto process_status; 9423dedbb5cSJason Baron 943743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 944743c69e7SNeilBrown status = -ENOBUFS; 945743c69e7SNeilBrown 946f279cd00SJason Baron if (sent > 0 || status == 0) { 947f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 948f279cd00SJason Baron if (sent >= req->rq_slen) 949262965f5SChuck Lever return 0; 950262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */ 951262965f5SChuck Lever status = -EAGAIN; 9522199700fSTrond Myklebust } 953262965f5SChuck Lever 9543dedbb5cSJason Baron process_status: 955262965f5SChuck Lever switch (status) { 956fba91afbSTrond Myklebust case -ENOTSOCK: 957fba91afbSTrond Myklebust status = -ENOTCONN; 958fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 959fba91afbSTrond Myklebust break; 960b6ddf64fSTrond Myklebust case -EAGAIN: 961c544577dSTrond Myklebust status = xs_nospace(req); 962b6ddf64fSTrond Myklebust break; 963262965f5SChuck Lever case -ENETUNREACH: 9643601c4a9STrond Myklebust case -ENOBUFS: 965262965f5SChuck Lever case -EPIPE: 966262965f5SChuck Lever case -ECONNREFUSED: 9673dedbb5cSJason Baron case -EPERM: 968262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message 969262965f5SChuck Lever * prompts ECONNREFUSED. */ 97013331a55STrond Myklebust break; 97113331a55STrond Myklebust default: 97213331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 97313331a55STrond Myklebust -status); 974262965f5SChuck Lever } 9755fe46e9dSBian Naimeng 976262965f5SChuck Lever return status; 977262965f5SChuck Lever } 978262965f5SChuck Lever 979e06799f9STrond Myklebust /** 980262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket 98150f484e2STrond Myklebust * @req: pointer to RPC request 982262965f5SChuck Lever * 983262965f5SChuck Lever * Return values: 984262965f5SChuck Lever * 0: The request has been sent 985262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to 986262965f5SChuck Lever * complete the request 987262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 98825985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 989262965f5SChuck Lever * 990262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up 991262965f5SChuck Lever * if sendmsg is not able to make progress? 992262965f5SChuck Lever */ 993adfa7144STrond Myklebust static int xs_tcp_send_request(struct rpc_rqst *req) 994262965f5SChuck Lever { 995262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 996ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 997262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 9987e3d3620STrond Myklebust rpc_fraghdr rm = xs_stream_record_marker(xdr); 9997e3d3620STrond Myklebust unsigned int msglen = rm ? req->rq_slen + sizeof(rm) : req->rq_slen; 10009e55eef4SChuck Lever struct msghdr msg = { 10019e55eef4SChuck Lever .msg_flags = XS_SENDMSG_FLAGS, 10029e55eef4SChuck Lever }; 10039ffadfbcSTrond Myklebust bool vm_wait = false; 10043f649ab7SKees Cook unsigned int sent; 1005b595bb15SChuck Lever int status; 1006a246b010SChuck Lever 10074cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 10084cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 10094cd34e7cSTrond Myklebust if (transport->sock != NULL) 10104cd34e7cSTrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR); 10114cd34e7cSTrond Myklebust return -ENOTCONN; 10124cd34e7cSTrond Myklebust } 10134cd34e7cSTrond Myklebust 1014262965f5SChuck Lever xs_pktdump("packet data:", 1015262965f5SChuck Lever req->rq_svec->iov_base, 1016262965f5SChuck Lever req->rq_svec->iov_len); 1017a246b010SChuck Lever 10187196dbb0STrond Myklebust if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state)) 10197196dbb0STrond Myklebust xs_tcp_set_socket_timeouts(xprt, transport->sock); 10207196dbb0STrond Myklebust 1021a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful 1022a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have 1023262965f5SChuck Lever * called sendmsg(). */ 102478215759SChuck Lever req->rq_xtime = ktime_get(); 1025d737e5d4STrond Myklebust tcp_sock_set_cork(transport->inet, true); 1026a246b010SChuck Lever while (1) { 10279e55eef4SChuck Lever status = xprt_sock_sendmsg(transport->sock, &msg, xdr, 10287e3d3620STrond Myklebust transport->xmit.offset, rm, &sent); 1029a246b010SChuck Lever 1030262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n", 10316c7a64e5STrond Myklebust xdr->len - transport->xmit.offset, status); 1032262965f5SChuck Lever 1033a246b010SChuck Lever /* If we've sent the entire packet, immediately 1034a246b010SChuck Lever * reset the count of bytes sent. */ 10356c7a64e5STrond Myklebust transport->xmit.offset += sent; 10366c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 10377e3d3620STrond Myklebust if (likely(req->rq_bytes_sent >= msglen)) { 10386c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 10396c7a64e5STrond Myklebust transport->xmit.offset = 0; 1040d737e5d4STrond Myklebust if (atomic_long_read(&xprt->xmit_queuelen) == 1) 1041d737e5d4STrond Myklebust tcp_sock_set_cork(transport->inet, false); 1042a246b010SChuck Lever return 0; 1043a246b010SChuck Lever } 1044262965f5SChuck Lever 10459ffadfbcSTrond Myklebust WARN_ON_ONCE(sent == 0 && status == 0); 10469ffadfbcSTrond Myklebust 10479ffadfbcSTrond Myklebust if (status == -EAGAIN ) { 10489ffadfbcSTrond Myklebust /* 10499ffadfbcSTrond Myklebust * Return EAGAIN if we're sure we're hitting the 10509ffadfbcSTrond Myklebust * socket send buffer limits. 10519ffadfbcSTrond Myklebust */ 10529ffadfbcSTrond Myklebust if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) 10539ffadfbcSTrond Myklebust break; 10549ffadfbcSTrond Myklebust /* 10559ffadfbcSTrond Myklebust * Did we hit a memory allocation failure? 10569ffadfbcSTrond Myklebust */ 10579ffadfbcSTrond Myklebust if (sent == 0) { 10589ffadfbcSTrond Myklebust status = -ENOBUFS; 10599ffadfbcSTrond Myklebust if (vm_wait) 10609ffadfbcSTrond Myklebust break; 10619ffadfbcSTrond Myklebust /* Retry, knowing now that we're below the 10629ffadfbcSTrond Myklebust * socket send buffer limit 10639ffadfbcSTrond Myklebust */ 10649ffadfbcSTrond Myklebust vm_wait = true; 10659ffadfbcSTrond Myklebust } 10669ffadfbcSTrond Myklebust continue; 10679ffadfbcSTrond Myklebust } 1068f580dd04STrond Myklebust if (status < 0) 1069f580dd04STrond Myklebust break; 10709ffadfbcSTrond Myklebust vm_wait = false; 1071a246b010SChuck Lever } 1072a246b010SChuck Lever 1073262965f5SChuck Lever switch (status) { 1074fba91afbSTrond Myklebust case -ENOTSOCK: 1075fba91afbSTrond Myklebust status = -ENOTCONN; 1076fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 1077fba91afbSTrond Myklebust break; 1078262965f5SChuck Lever case -EAGAIN: 1079c544577dSTrond Myklebust status = xs_nospace(req); 1080262965f5SChuck Lever break; 1081262965f5SChuck Lever case -ECONNRESET: 1082262965f5SChuck Lever case -ECONNREFUSED: 1083262965f5SChuck Lever case -ENOTCONN: 10843913c78cSTrond Myklebust case -EADDRINUSE: 1085b5872f0cSTrond Myklebust case -ENOBUFS: 1086b9d2bb2eSTrond Myklebust case -EPIPE: 108713331a55STrond Myklebust break; 108813331a55STrond Myklebust default: 108913331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 109013331a55STrond Myklebust -status); 1091a246b010SChuck Lever } 10925fe46e9dSBian Naimeng 1093a246b010SChuck Lever return status; 1094a246b010SChuck Lever } 1095a246b010SChuck Lever 10962a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) 10972a9e1cfaSTrond Myklebust { 10982a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready; 10992a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change; 11002a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space; 11012118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report; 11022a9e1cfaSTrond Myklebust } 11032a9e1cfaSTrond Myklebust 11042a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) 11052a9e1cfaSTrond Myklebust { 11062a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready; 11072a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change; 11082a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space; 11092118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report; 11102118071dSTrond Myklebust } 11112118071dSTrond Myklebust 111242d42a5bSTrond Myklebust static void xs_sock_reset_state_flags(struct rpc_xprt *xprt) 111342d42a5bSTrond Myklebust { 111442d42a5bSTrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 111542d42a5bSTrond Myklebust 111642d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 11174f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state); 11184f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state); 11194f8943f8STrond Myklebust clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state); 11204f8943f8STrond Myklebust } 11214f8943f8STrond Myklebust 11224f8943f8STrond Myklebust static void xs_run_error_worker(struct sock_xprt *transport, unsigned int nr) 11234f8943f8STrond Myklebust { 11244f8943f8STrond Myklebust set_bit(nr, &transport->sock_state); 11254f8943f8STrond Myklebust queue_work(xprtiod_workqueue, &transport->error_worker); 112642d42a5bSTrond Myklebust } 112742d42a5bSTrond Myklebust 1128b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt) 1129b70ae915STrond Myklebust { 1130b70ae915STrond Myklebust smp_mb__before_atomic(); 1131b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 1132b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 113342d42a5bSTrond Myklebust xs_sock_reset_state_flags(xprt); 1134b70ae915STrond Myklebust smp_mb__after_atomic(); 1135b70ae915STrond Myklebust } 1136b70ae915STrond Myklebust 11372118071dSTrond Myklebust /** 11382118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors 11392118071dSTrond Myklebust * @sk: socket 11402118071dSTrond Myklebust * 11412118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task 11422118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err. 11432118071dSTrond Myklebust */ 11442118071dSTrond Myklebust static void xs_error_report(struct sock *sk) 11452118071dSTrond Myklebust { 11464f8943f8STrond Myklebust struct sock_xprt *transport; 11472118071dSTrond Myklebust struct rpc_xprt *xprt; 11482118071dSTrond Myklebust 11492118071dSTrond Myklebust read_lock_bh(&sk->sk_callback_lock); 11502118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 11512118071dSTrond Myklebust goto out; 11522118071dSTrond Myklebust 11534f8943f8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 1154af84537dSBenjamin Coddington transport->xprt_err = -sk->sk_err; 1155af84537dSBenjamin Coddington if (transport->xprt_err == 0) 11562118071dSTrond Myklebust goto out; 11572118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n", 1158af84537dSBenjamin Coddington xprt, -transport->xprt_err); 1159af84537dSBenjamin Coddington trace_rpc_socket_error(xprt, sk->sk_socket, transport->xprt_err); 1160af84537dSBenjamin Coddington 1161af84537dSBenjamin Coddington /* barrier ensures xprt_err is set before XPRT_SOCK_WAKE_ERROR */ 1162af84537dSBenjamin Coddington smp_mb__before_atomic(); 11634f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_ERROR); 11642118071dSTrond Myklebust out: 11652118071dSTrond Myklebust read_unlock_bh(&sk->sk_callback_lock); 11662a9e1cfaSTrond Myklebust } 11672a9e1cfaSTrond Myklebust 1168fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport) 1169a246b010SChuck Lever { 1170ee0ac0c2SChuck Lever struct socket *sock = transport->sock; 1171ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 11726cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 1173a73881c9STrond Myklebust struct file *filp = transport->file; 1174a246b010SChuck Lever 1175fe315e76SChuck Lever if (sk == NULL) 1176fe315e76SChuck Lever return; 11779903cd1cSChuck Lever 1178264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper)) 1179264d1df3SJeff Layton sk_clear_memalloc(sk); 1180264d1df3SJeff Layton 118109939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 118209939204STrond Myklebust 1183edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 1184a246b010SChuck Lever write_lock_bh(&sk->sk_callback_lock); 1185ee0ac0c2SChuck Lever transport->inet = NULL; 1186ee0ac0c2SChuck Lever transport->sock = NULL; 1187a73881c9STrond Myklebust transport->file = NULL; 1188a246b010SChuck Lever 1189a246b010SChuck Lever sk->sk_user_data = NULL; 11902a9e1cfaSTrond Myklebust 11912a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk); 11920c78789eSTrond Myklebust xprt_clear_connected(xprt); 1193a246b010SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 11946cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt); 1195ae053551STrond Myklebust /* Reset stream record info */ 1196ae053551STrond Myklebust xs_stream_reset_connect(transport); 1197edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 1198a246b010SChuck Lever 11996cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock); 1200a73881c9STrond Myklebust fput(filp); 12010445f92cSTrond Myklebust 12020445f92cSTrond Myklebust xprt_disconnect_done(xprt); 1203fe315e76SChuck Lever } 1204fe315e76SChuck Lever 1205fe315e76SChuck Lever /** 1206fe315e76SChuck Lever * xs_close - close a socket 1207fe315e76SChuck Lever * @xprt: transport 1208fe315e76SChuck Lever * 1209fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains 1210fe315e76SChuck Lever * on the server we want to save. 1211f75e6745STrond Myklebust * 1212f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with 1213f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer. 1214fe315e76SChuck Lever */ 1215fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt) 1216fe315e76SChuck Lever { 1217fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1218fe315e76SChuck Lever 1219fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt); 1220fe315e76SChuck Lever 1221fe315e76SChuck Lever xs_reset_transport(transport); 122261d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 1223a246b010SChuck Lever } 1224a246b010SChuck Lever 12254a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt) 12264a068258SChuck Lever { 12274a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n", 12284a068258SChuck Lever xprt); 12294a068258SChuck Lever xprt_disconnect_done(xprt); 12304a068258SChuck Lever } 12314a068258SChuck Lever 1232315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt) 1233315f3812SKinglong Mee { 1234315f3812SKinglong Mee xs_free_peer_addresses(xprt); 1235315f3812SKinglong Mee xprt_free(xprt); 1236315f3812SKinglong Mee } 1237315f3812SKinglong Mee 12389903cd1cSChuck Lever /** 12399903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport 12409903cd1cSChuck Lever * @xprt: doomed transport 12419903cd1cSChuck Lever * 12429903cd1cSChuck Lever */ 12439903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt) 1244a246b010SChuck Lever { 124503c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt, 124603c78827STrond Myklebust struct sock_xprt, xprt); 12479903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt); 12489903cd1cSChuck Lever 124903c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker); 1250a1311d87STrond Myklebust xs_close(xprt); 1251edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker); 1252b5e92419STrond Myklebust cancel_work_sync(&transport->error_worker); 1253315f3812SKinglong Mee xs_xprt_free(xprt); 1254a1311d87STrond Myklebust module_put(THIS_MODULE); 1255a246b010SChuck Lever } 1256a246b010SChuck Lever 12579903cd1cSChuck Lever /** 1258f9b2ee71STrond Myklebust * xs_udp_data_read_skb - receive callback for UDP sockets 1259f9b2ee71STrond Myklebust * @xprt: transport 1260f9b2ee71STrond Myklebust * @sk: socket 1261f9b2ee71STrond Myklebust * @skb: skbuff 12629903cd1cSChuck Lever * 1263a246b010SChuck Lever */ 1264f9b2ee71STrond Myklebust static void xs_udp_data_read_skb(struct rpc_xprt *xprt, 1265f9b2ee71STrond Myklebust struct sock *sk, 1266f9b2ee71STrond Myklebust struct sk_buff *skb) 1267a246b010SChuck Lever { 1268a246b010SChuck Lever struct rpc_task *task; 1269a246b010SChuck Lever struct rpc_rqst *rovr; 1270f9b2ee71STrond Myklebust int repsize, copied; 1271d8ed029dSAlexey Dobriyan u32 _xid; 1272d8ed029dSAlexey Dobriyan __be32 *xp; 1273a246b010SChuck Lever 12741da8c681SWillem de Bruijn repsize = skb->len; 1275a246b010SChuck Lever if (repsize < 4) { 12769903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize); 1277f9b2ee71STrond Myklebust return; 1278a246b010SChuck Lever } 1279a246b010SChuck Lever 1280a246b010SChuck Lever /* Copy the XID from the skb... */ 12811da8c681SWillem de Bruijn xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid); 1282a246b010SChuck Lever if (xp == NULL) 1283f9b2ee71STrond Myklebust return; 1284a246b010SChuck Lever 1285a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */ 128675c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1287a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 1288a246b010SChuck Lever if (!rovr) 1289a246b010SChuck Lever goto out_unlock; 1290729749bbSTrond Myklebust xprt_pin_rqst(rovr); 1291ecd465eeSChuck Lever xprt_update_rtt(rovr->rq_task); 129275c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1293a246b010SChuck Lever task = rovr->rq_task; 1294a246b010SChuck Lever 1295a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize) 1296a246b010SChuck Lever copied = repsize; 1297a246b010SChuck Lever 1298a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */ 12991781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) { 130075c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13010afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INERRORS); 1302729749bbSTrond Myklebust goto out_unpin; 13031781f7f5SHerbert Xu } 13041781f7f5SHerbert Xu 1305a246b010SChuck Lever 1306b5e92419STrond Myklebust spin_lock(&xprt->transport_lock); 13076a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied); 1308b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock); 130975c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13101570c1e4SChuck Lever xprt_complete_rqst(task, copied); 13110afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS); 1312729749bbSTrond Myklebust out_unpin: 1313729749bbSTrond Myklebust xprt_unpin_rqst(rovr); 1314a246b010SChuck Lever out_unlock: 131575c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1316f9b2ee71STrond Myklebust } 1317f9b2ee71STrond Myklebust 1318f9b2ee71STrond Myklebust static void xs_udp_data_receive(struct sock_xprt *transport) 1319f9b2ee71STrond Myklebust { 1320f9b2ee71STrond Myklebust struct sk_buff *skb; 1321f9b2ee71STrond Myklebust struct sock *sk; 1322f9b2ee71STrond Myklebust int err; 1323f9b2ee71STrond Myklebust 1324f9b2ee71STrond Myklebust mutex_lock(&transport->recv_mutex); 1325f9b2ee71STrond Myklebust sk = transport->inet; 1326f9b2ee71STrond Myklebust if (sk == NULL) 1327f9b2ee71STrond Myklebust goto out; 1328f9b2ee71STrond Myklebust for (;;) { 13297c13f97fSPaolo Abeni skb = skb_recv_udp(sk, 0, 1, &err); 13304f546149STrond Myklebust if (skb == NULL) 13314f546149STrond Myklebust break; 1332f9b2ee71STrond Myklebust xs_udp_data_read_skb(&transport->xprt, sk, skb); 1333850cbaddSPaolo Abeni consume_skb(skb); 13340af3442aSTrond Myklebust cond_resched(); 1335f9b2ee71STrond Myklebust } 13360ffe86f4STrond Myklebust xs_poll_check_readable(transport); 1337a246b010SChuck Lever out: 1338f9b2ee71STrond Myklebust mutex_unlock(&transport->recv_mutex); 1339f9b2ee71STrond Myklebust } 1340f9b2ee71STrond Myklebust 1341f9b2ee71STrond Myklebust static void xs_udp_data_receive_workfn(struct work_struct *work) 1342f9b2ee71STrond Myklebust { 1343f9b2ee71STrond Myklebust struct sock_xprt *transport = 1344f9b2ee71STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1345a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 1346a1231fdaSTrond Myklebust 1347f9b2ee71STrond Myklebust xs_udp_data_receive(transport); 1348a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 1349f9b2ee71STrond Myklebust } 1350f9b2ee71STrond Myklebust 1351f9b2ee71STrond Myklebust /** 1352f9b2ee71STrond Myklebust * xs_data_ready - "data ready" callback for UDP sockets 1353f9b2ee71STrond Myklebust * @sk: socket with data to read 1354f9b2ee71STrond Myklebust * 1355f9b2ee71STrond Myklebust */ 1356f9b2ee71STrond Myklebust static void xs_data_ready(struct sock *sk) 1357f9b2ee71STrond Myklebust { 1358f9b2ee71STrond Myklebust struct rpc_xprt *xprt; 1359f9b2ee71STrond Myklebust 1360f9b2ee71STrond Myklebust read_lock_bh(&sk->sk_callback_lock); 1361f9b2ee71STrond Myklebust dprintk("RPC: xs_data_ready...\n"); 1362f9b2ee71STrond Myklebust xprt = xprt_from_sock(sk); 1363f9b2ee71STrond Myklebust if (xprt != NULL) { 1364f9b2ee71STrond Myklebust struct sock_xprt *transport = container_of(xprt, 1365f9b2ee71STrond Myklebust struct sock_xprt, xprt); 13665157b956STrond Myklebust transport->old_data_ready(sk); 13675157b956STrond Myklebust /* Any data means we had a useful conversation, so 13685157b956STrond Myklebust * then we don't need to delay the next reconnect 13695157b956STrond Myklebust */ 13705157b956STrond Myklebust if (xprt->reestablish_timeout) 13715157b956STrond Myklebust xprt->reestablish_timeout = 0; 137242d42a5bSTrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 137340a5f1b1STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 1374f9b2ee71STrond Myklebust } 1375f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1376a246b010SChuck Lever } 1377a246b010SChuck Lever 1378a519fc7aSTrond Myklebust /* 1379a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending 1380a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT 1381a519fc7aSTrond Myklebust */ 1382a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt) 1383a519fc7aSTrond Myklebust { 1384a519fc7aSTrond Myklebust xprt_force_disconnect(xprt); 1385a519fc7aSTrond Myklebust } 1386a519fc7aSTrond Myklebust 13879e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 13886b26cc8cSChuck Lever static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt) 13896b26cc8cSChuck Lever { 13906b26cc8cSChuck Lever return PAGE_SIZE; 13916b26cc8cSChuck Lever } 13929e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 139344b98efdSRicardo Labiaga 13949903cd1cSChuck Lever /** 13959903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes 13969903cd1cSChuck Lever * @sk: socket whose state has changed 13979903cd1cSChuck Lever * 13989903cd1cSChuck Lever */ 13999903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk) 1400a246b010SChuck Lever { 1401a246b010SChuck Lever struct rpc_xprt *xprt; 14020fdea1e8STrond Myklebust struct sock_xprt *transport; 1403a246b010SChuck Lever 1404f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1405a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk))) 1406a246b010SChuck Lever goto out; 14079903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt); 1408669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n", 1409a246b010SChuck Lever sk->sk_state, xprt_connected(xprt), 1410a246b010SChuck Lever sock_flag(sk, SOCK_DEAD), 1411669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED), 1412669502ffSAndy Chittenden sk->sk_shutdown); 1413a246b010SChuck Lever 14140fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 141540b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket); 1416a246b010SChuck Lever switch (sk->sk_state) { 1417a246b010SChuck Lever case TCP_ESTABLISHED: 1418a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) { 14198b71798cSTrond Myklebust xprt->connect_cookie++; 14200fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 14210fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 142251971139SChuck Lever 14233968a8a5SChuck Lever xprt->stat.connect_count++; 14243968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 14253968a8a5SChuck Lever xprt->stat.connect_start; 14264f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_PENDING); 1427a246b010SChuck Lever } 1428a246b010SChuck Lever break; 14293b948ae5STrond Myklebust case TCP_FIN_WAIT1: 14303b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */ 14317c1d71cfSTrond Myklebust xprt->connect_cookie++; 1432663b8858STrond Myklebust xprt->reestablish_timeout = 0; 14333b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14344e857c58SPeter Zijlstra smp_mb__before_atomic(); 14353b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1436ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 14374e857c58SPeter Zijlstra smp_mb__after_atomic(); 1438a246b010SChuck Lever break; 1439632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT: 14403b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */ 14417c1d71cfSTrond Myklebust xprt->connect_cookie++; 1442d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 14434f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT); 1444df561f66SGustavo A. R. Silva fallthrough; 1445663b8858STrond Myklebust case TCP_CLOSING: 1446663b8858STrond Myklebust /* 1447663b8858STrond Myklebust * If the server closed down the connection, make sure that 1448663b8858STrond Myklebust * we back off before reconnecting 1449663b8858STrond Myklebust */ 1450663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 1451663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 14523b948ae5STrond Myklebust break; 14533b948ae5STrond Myklebust case TCP_LAST_ACK: 1454670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14554e857c58SPeter Zijlstra smp_mb__before_atomic(); 14563b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 14574e857c58SPeter Zijlstra smp_mb__after_atomic(); 14583b948ae5STrond Myklebust break; 14593b948ae5STrond Myklebust case TCP_CLOSE: 14600fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING, 14610fdea1e8STrond Myklebust &transport->sock_state)) 14620fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 14639b30889cSTrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 14649b30889cSTrond Myklebust /* Trigger the socket release */ 14654f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_DISCONNECT); 1466a246b010SChuck Lever } 1467a246b010SChuck Lever out: 1468f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1469a246b010SChuck Lever } 1470a246b010SChuck Lever 14711f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk) 14721f0fa154SIlpo Järvinen { 147313331a55STrond Myklebust struct socket_wq *wq; 14744f8943f8STrond Myklebust struct sock_xprt *transport; 14751f0fa154SIlpo Järvinen struct rpc_xprt *xprt; 14761f0fa154SIlpo Järvinen 147713331a55STrond Myklebust if (!sk->sk_socket) 14781f0fa154SIlpo Järvinen return; 147913331a55STrond Myklebust clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 14801f0fa154SIlpo Järvinen 14811f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk)))) 14821f0fa154SIlpo Järvinen return; 14834f8943f8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 148413331a55STrond Myklebust rcu_read_lock(); 148513331a55STrond Myklebust wq = rcu_dereference(sk->sk_wq); 148613331a55STrond Myklebust if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0) 148713331a55STrond Myklebust goto out; 14881f0fa154SIlpo Järvinen 14894f8943f8STrond Myklebust xs_run_error_worker(transport, XPRT_SOCK_WAKE_WRITE); 1490c544577dSTrond Myklebust sk->sk_write_pending--; 149113331a55STrond Myklebust out: 149213331a55STrond Myklebust rcu_read_unlock(); 14931f0fa154SIlpo Järvinen } 14941f0fa154SIlpo Järvinen 14952a9e1cfaSTrond Myklebust /** 1496c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space 1497c7b2cae8SChuck Lever * becomes available 14989903cd1cSChuck Lever * @sk: socket whose state has changed 14999903cd1cSChuck Lever * 1500a246b010SChuck Lever * Called when more output buffer space is available for this socket. 1501a246b010SChuck Lever * We try not to wake our writers until they can make "significant" 1502c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1503a246b010SChuck Lever * with a bunch of small requests. 1504a246b010SChuck Lever */ 1505c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk) 1506a246b010SChuck Lever { 1507f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1508c7b2cae8SChuck Lever 1509c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */ 15101f0fa154SIlpo Järvinen if (sock_writeable(sk)) 15111f0fa154SIlpo Järvinen xs_write_space(sk); 1512c7b2cae8SChuck Lever 1513f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1514c7b2cae8SChuck Lever } 1515c7b2cae8SChuck Lever 1516c7b2cae8SChuck Lever /** 1517c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space 1518c7b2cae8SChuck Lever * becomes available 1519c7b2cae8SChuck Lever * @sk: socket whose state has changed 1520c7b2cae8SChuck Lever * 1521c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket. 1522c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant" 1523c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1524c7b2cae8SChuck Lever * with a bunch of small requests. 1525c7b2cae8SChuck Lever */ 1526c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk) 1527c7b2cae8SChuck Lever { 1528f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1529c7b2cae8SChuck Lever 1530c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */ 153164dc6130SEric Dumazet if (sk_stream_is_writeable(sk)) 15321f0fa154SIlpo Järvinen xs_write_space(sk); 1533c7b2cae8SChuck Lever 1534f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1535a246b010SChuck Lever } 1536a246b010SChuck Lever 1537470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) 1538a246b010SChuck Lever { 1539ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1540ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 1541a246b010SChuck Lever 15427c6e066eSChuck Lever if (transport->rcvsize) { 1543a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 15447c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2; 1545a246b010SChuck Lever } 15467c6e066eSChuck Lever if (transport->sndsize) { 1547a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 15487c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; 1549a246b010SChuck Lever sk->sk_write_space(sk); 1550a246b010SChuck Lever } 1551a246b010SChuck Lever } 1552a246b010SChuck Lever 155343118c29SChuck Lever /** 1554470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits 155543118c29SChuck Lever * @xprt: generic transport 1556470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes 1557470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes 155843118c29SChuck Lever * 1559470056c2SChuck Lever * Set socket send and receive buffer size limits. 156043118c29SChuck Lever */ 1561470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize) 156243118c29SChuck Lever { 15637c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 15647c6e066eSChuck Lever 15657c6e066eSChuck Lever transport->sndsize = 0; 1566470056c2SChuck Lever if (sndsize) 15677c6e066eSChuck Lever transport->sndsize = sndsize + 1024; 15687c6e066eSChuck Lever transport->rcvsize = 0; 1569470056c2SChuck Lever if (rcvsize) 15707c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024; 1571470056c2SChuck Lever 1572470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 157343118c29SChuck Lever } 157443118c29SChuck Lever 157546c0ee8bSChuck Lever /** 157646c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport 1577acf0a39fSChuck Lever * @xprt: controlling transport 157846c0ee8bSChuck Lever * @task: task that timed out 157946c0ee8bSChuck Lever * 158046c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred. 158146c0ee8bSChuck Lever */ 15826a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task) 158346c0ee8bSChuck Lever { 1584b5e92419STrond Myklebust spin_lock(&xprt->transport_lock); 15856a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT); 1586b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock); 158746c0ee8bSChuck Lever } 158846c0ee8bSChuck Lever 1589826799e6SJ. Bruce Fields static int xs_get_random_port(void) 1590b85d8806SChuck Lever { 1591826799e6SJ. Bruce Fields unsigned short min = xprt_min_resvport, max = xprt_max_resvport; 1592826799e6SJ. Bruce Fields unsigned short range; 1593826799e6SJ. Bruce Fields unsigned short rand; 1594826799e6SJ. Bruce Fields 1595826799e6SJ. Bruce Fields if (max < min) 1596826799e6SJ. Bruce Fields return -EADDRINUSE; 1597826799e6SJ. Bruce Fields range = max - min + 1; 1598826799e6SJ. Bruce Fields rand = (unsigned short) prandom_u32() % range; 1599826799e6SJ. Bruce Fields return rand + min; 1600b85d8806SChuck Lever } 1601b85d8806SChuck Lever 16024dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock) 16034dda9c8aSTrond Myklebust { 16044dda9c8aSTrond Myklebust struct sockaddr_storage buf; 16054dda9c8aSTrond Myklebust unsigned short port = 0; 16064dda9c8aSTrond Myklebust 16079b2c45d4SDenys Vlasenko if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0) 16084dda9c8aSTrond Myklebust goto out; 16094dda9c8aSTrond Myklebust switch (buf.ss_family) { 16104dda9c8aSTrond Myklebust case AF_INET6: 16114dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port); 16124dda9c8aSTrond Myklebust break; 16134dda9c8aSTrond Myklebust case AF_INET: 16144dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port); 16154dda9c8aSTrond Myklebust } 16164dda9c8aSTrond Myklebust out: 16174dda9c8aSTrond Myklebust return port; 16184dda9c8aSTrond Myklebust } 16194dda9c8aSTrond Myklebust 16204dda9c8aSTrond Myklebust /** 162192200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address 162292200412SChuck Lever * @xprt: generic transport 162392200412SChuck Lever * @port: new port number 162492200412SChuck Lever * 162592200412SChuck Lever */ 162692200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) 162792200412SChuck Lever { 162892200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port); 1629c4efcb1dSChuck Lever 16309dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port); 16319dc3b095SChuck Lever xs_update_peer_port(xprt); 163292200412SChuck Lever } 163392200412SChuck Lever 16344dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock) 16354dda9c8aSTrond Myklebust { 1636e6237b6fSTrond Myklebust if (transport->srcport == 0 && transport->xprt.reuseport) 16374dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock); 16384dda9c8aSTrond Myklebust } 16394dda9c8aSTrond Myklebust 1640826799e6SJ. Bruce Fields static int xs_get_srcport(struct sock_xprt *transport) 164167a391d7STrond Myklebust { 1642826799e6SJ. Bruce Fields int port = transport->srcport; 164367a391d7STrond Myklebust 164467a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport) 164567a391d7STrond Myklebust port = xs_get_random_port(); 164667a391d7STrond Myklebust return port; 164767a391d7STrond Myklebust } 164867a391d7STrond Myklebust 1649baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) 165067a391d7STrond Myklebust { 1651fbfffbd5SChuck Lever if (transport->srcport != 0) 1652fbfffbd5SChuck Lever transport->srcport = 0; 165367a391d7STrond Myklebust if (!transport->xprt.resvport) 165467a391d7STrond Myklebust return 0; 165567a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport) 165667a391d7STrond Myklebust return xprt_max_resvport; 165767a391d7STrond Myklebust return --port; 165867a391d7STrond Myklebust } 1659beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock) 1660a246b010SChuck Lever { 1661beb59b68SPavel Emelyanov struct sockaddr_storage myaddr; 166267a391d7STrond Myklebust int err, nloop = 0; 1663826799e6SJ. Bruce Fields int port = xs_get_srcport(transport); 166467a391d7STrond Myklebust unsigned short last; 1665a246b010SChuck Lever 16660f7a622cSChris Perl /* 16670f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 && 16680f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local 16690f7a622cSChris Perl * port selection happen implicitly when the socket is used 16700f7a622cSChris Perl * (for example at connect time). 16710f7a622cSChris Perl * 16720f7a622cSChris Perl * This ensures that we can continue to establish TCP 16730f7a622cSChris Perl * connections even when all local ephemeral ports are already 16740f7a622cSChris Perl * a part of some TCP connection. This makes no difference 167512b20ce3SBhaskar Chowdhury * for UDP sockets, but also doesn't harm them. 16760f7a622cSChris Perl * 16770f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 && 16780f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will 16790f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed. 16800f7a622cSChris Perl */ 1681826799e6SJ. Bruce Fields if (port <= 0) 1682826799e6SJ. Bruce Fields return port; 16830f7a622cSChris Perl 1684beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen); 1685a246b010SChuck Lever do { 1686beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port); 1687e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr, 1688beb59b68SPavel Emelyanov transport->xprt.addrlen); 1689a246b010SChuck Lever if (err == 0) { 1690fbfffbd5SChuck Lever transport->srcport = port; 1691d3bc9a1dSFrank van Maarseveen break; 1692a246b010SChuck Lever } 169367a391d7STrond Myklebust last = port; 1694baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port); 169567a391d7STrond Myklebust if (port > last) 169667a391d7STrond Myklebust nloop++; 169767a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2); 1698beb59b68SPavel Emelyanov 16994232e863SChuck Lever if (myaddr.ss_family == AF_INET) 1700beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__, 1701beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr, 1702beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err); 1703beb59b68SPavel Emelyanov else 1704beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__, 1705beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr, 17067dc753f0SChuck Lever port, err ? "failed" : "ok", err); 1707a246b010SChuck Lever return err; 1708a246b010SChuck Lever } 1709a246b010SChuck Lever 1710176e21eeSChuck Lever /* 1711176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets 1712176e21eeSChuck Lever */ 1713176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task) 1714176e21eeSChuck Lever { 1715fb43d172STrond Myklebust xprt_set_bound(task->tk_xprt); 1716176e21eeSChuck Lever } 1717176e21eeSChuck Lever 1718176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) 1719176e21eeSChuck Lever { 1720176e21eeSChuck Lever } 1721a246b010SChuck Lever 1722ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC 1723ed07536eSPeter Zijlstra static struct lock_class_key xs_key[2]; 1724ed07536eSPeter Zijlstra static struct lock_class_key xs_slock_key[2]; 1725ed07536eSPeter Zijlstra 1726176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1727176e21eeSChuck Lever { 1728176e21eeSChuck Lever struct sock *sk = sock->sk; 1729176e21eeSChuck Lever 1730176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC", 1731176e21eeSChuck Lever &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]); 1732176e21eeSChuck Lever } 1733176e21eeSChuck Lever 17348945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 1735ed07536eSPeter Zijlstra { 1736ed07536eSPeter Zijlstra struct sock *sk = sock->sk; 17378945ee5eSChuck Lever 17388945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC", 17398945ee5eSChuck Lever &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]); 1740ed07536eSPeter Zijlstra } 17418945ee5eSChuck Lever 17428945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 17438945ee5eSChuck Lever { 17448945ee5eSChuck Lever struct sock *sk = sock->sk; 17458945ee5eSChuck Lever 17468945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC", 17478945ee5eSChuck Lever &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]); 1748ed07536eSPeter Zijlstra } 17496bc9638aSPavel Emelyanov 17506bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 17516bc9638aSPavel Emelyanov { 1752fafc4e1eSHannes Frederic Sowa if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk))) 17531b7a1819SWeston Andros Adamson return; 17541b7a1819SWeston Andros Adamson 17554232e863SChuck Lever switch (family) { 1756176e21eeSChuck Lever case AF_LOCAL: 1757176e21eeSChuck Lever xs_reclassify_socketu(sock); 1758176e21eeSChuck Lever break; 17594232e863SChuck Lever case AF_INET: 17606bc9638aSPavel Emelyanov xs_reclassify_socket4(sock); 17614232e863SChuck Lever break; 17624232e863SChuck Lever case AF_INET6: 17636bc9638aSPavel Emelyanov xs_reclassify_socket6(sock); 17644232e863SChuck Lever break; 17654232e863SChuck Lever } 17666bc9638aSPavel Emelyanov } 1767ed07536eSPeter Zijlstra #else 17686bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 17696bc9638aSPavel Emelyanov { 17706bc9638aSPavel Emelyanov } 1771ed07536eSPeter Zijlstra #endif 1772ed07536eSPeter Zijlstra 177393dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work) 177493dc41bdSNeilBrown { 177593dc41bdSNeilBrown } 177693dc41bdSNeilBrown 17776bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt, 17784dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type, 17794dda9c8aSTrond Myklebust int protocol, bool reuseport) 178022f79326SPavel Emelyanov { 1781a73881c9STrond Myklebust struct file *filp; 178222f79326SPavel Emelyanov struct socket *sock; 178322f79326SPavel Emelyanov int err; 178422f79326SPavel Emelyanov 17856bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1); 178622f79326SPavel Emelyanov if (err < 0) { 178722f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n", 178822f79326SPavel Emelyanov protocol, -err); 178922f79326SPavel Emelyanov goto out; 179022f79326SPavel Emelyanov } 17916bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock); 179222f79326SPavel Emelyanov 17934dda9c8aSTrond Myklebust if (reuseport) 1794fe31a326SChristoph Hellwig sock_set_reuseport(sock->sk); 17954dda9c8aSTrond Myklebust 17964cea288aSBen Hutchings err = xs_bind(transport, sock); 17974cea288aSBen Hutchings if (err) { 179822f79326SPavel Emelyanov sock_release(sock); 179922f79326SPavel Emelyanov goto out; 180022f79326SPavel Emelyanov } 180122f79326SPavel Emelyanov 1802a73881c9STrond Myklebust filp = sock_alloc_file(sock, O_NONBLOCK, NULL); 1803a73881c9STrond Myklebust if (IS_ERR(filp)) 1804a73881c9STrond Myklebust return ERR_CAST(filp); 1805a73881c9STrond Myklebust transport->file = filp; 1806a73881c9STrond Myklebust 180722f79326SPavel Emelyanov return sock; 180822f79326SPavel Emelyanov out: 180922f79326SPavel Emelyanov return ERR_PTR(err); 181022f79326SPavel Emelyanov } 181122f79326SPavel Emelyanov 1812176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt, 1813176e21eeSChuck Lever struct socket *sock) 1814176e21eeSChuck Lever { 1815176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1816176e21eeSChuck Lever xprt); 1817176e21eeSChuck Lever 1818176e21eeSChuck Lever if (!transport->inet) { 1819176e21eeSChuck Lever struct sock *sk = sock->sk; 1820176e21eeSChuck Lever 1821176e21eeSChuck Lever write_lock_bh(&sk->sk_callback_lock); 1822176e21eeSChuck Lever 1823176e21eeSChuck Lever xs_save_old_callbacks(transport, sk); 1824176e21eeSChuck Lever 1825176e21eeSChuck Lever sk->sk_user_data = xprt; 1826a2648094STrond Myklebust sk->sk_data_ready = xs_data_ready; 1827176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space; 1828b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 18292118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 1830176e21eeSChuck Lever 1831176e21eeSChuck Lever xprt_clear_connected(xprt); 1832176e21eeSChuck Lever 1833176e21eeSChuck Lever /* Reset to new socket */ 1834176e21eeSChuck Lever transport->sock = sock; 1835176e21eeSChuck Lever transport->inet = sk; 1836176e21eeSChuck Lever 1837176e21eeSChuck Lever write_unlock_bh(&sk->sk_callback_lock); 1838176e21eeSChuck Lever } 1839176e21eeSChuck Lever 1840ae053551STrond Myklebust xs_stream_start_connect(transport); 18416c7a64e5STrond Myklebust 1842176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0); 1843176e21eeSChuck Lever } 1844176e21eeSChuck Lever 1845176e21eeSChuck Lever /** 1846176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint 1847176e21eeSChuck Lever * @transport: socket transport to connect 1848176e21eeSChuck Lever */ 1849dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport) 1850176e21eeSChuck Lever { 1851176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt; 1852a73881c9STrond Myklebust struct file *filp; 1853176e21eeSChuck Lever struct socket *sock; 185468e9a246SColin Ian King int status; 1855176e21eeSChuck Lever 1856176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL, 1857176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1); 1858176e21eeSChuck Lever if (status < 0) { 1859176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL " 1860176e21eeSChuck Lever "transport socket (%d).\n", -status); 1861176e21eeSChuck Lever goto out; 1862176e21eeSChuck Lever } 1863d1358917SStefan Hajnoczi xs_reclassify_socket(AF_LOCAL, sock); 1864176e21eeSChuck Lever 1865a73881c9STrond Myklebust filp = sock_alloc_file(sock, O_NONBLOCK, NULL); 1866a73881c9STrond Myklebust if (IS_ERR(filp)) { 1867a73881c9STrond Myklebust status = PTR_ERR(filp); 1868a73881c9STrond Myklebust goto out; 1869a73881c9STrond Myklebust } 1870a73881c9STrond Myklebust transport->file = filp; 1871a73881c9STrond Myklebust 1872176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n", 1873176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1874176e21eeSChuck Lever 1875176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock); 187640b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 1877176e21eeSChuck Lever switch (status) { 1878176e21eeSChuck Lever case 0: 1879176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n", 1880176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 18813968a8a5SChuck Lever xprt->stat.connect_count++; 18823968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 18833968a8a5SChuck Lever xprt->stat.connect_start; 1884176e21eeSChuck Lever xprt_set_connected(xprt); 188593f479d3SGustavo A. R. Silva break; 18863601c4a9STrond Myklebust case -ENOBUFS: 1887176e21eeSChuck Lever break; 1888176e21eeSChuck Lever case -ENOENT: 1889176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n", 1890176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1891176e21eeSChuck Lever break; 18924a20a988STrond Myklebust case -ECONNREFUSED: 18934a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n", 18944a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 18954a20a988STrond Myklebust break; 1896176e21eeSChuck Lever default: 1897176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n", 1898176e21eeSChuck Lever __func__, -status, 1899176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 1900176e21eeSChuck Lever } 1901176e21eeSChuck Lever 1902176e21eeSChuck Lever out: 1903176e21eeSChuck Lever xprt_clear_connecting(xprt); 1904176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status); 1905dc107402SJ. Bruce Fields return status; 1906dc107402SJ. Bruce Fields } 1907dc107402SJ. Bruce Fields 1908b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) 1909dc107402SJ. Bruce Fields { 1910dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1911dc107402SJ. Bruce Fields int ret; 1912dc107402SJ. Bruce Fields 1913dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) { 1914dc107402SJ. Bruce Fields /* 1915dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the 1916dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc 1917dc107402SJ. Bruce Fields * call. Thus we connect synchronously. 1918dc107402SJ. Bruce Fields * 1919dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls, 1920dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to 1921dc107402SJ. Bruce Fields * connect. 1922dc107402SJ. Bruce Fields */ 19235ad64b36STrond Myklebust task->tk_rpc_status = -ENOTCONN; 1924dc107402SJ. Bruce Fields rpc_exit(task, -ENOTCONN); 1925dc107402SJ. Bruce Fields return; 1926dc107402SJ. Bruce Fields } 1927dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport); 1928dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task)) 1929dc107402SJ. Bruce Fields msleep_interruptible(15000); 1930176e21eeSChuck Lever } 1931176e21eeSChuck Lever 19323c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP) 1933d6e971d8SJeff Layton /* 1934d6e971d8SJeff Layton * Note that this should be called with XPRT_LOCKED held (or when we otherwise 1935d6e971d8SJeff Layton * know that we have exclusive access to the socket), to guard against 1936d6e971d8SJeff Layton * races with xs_reset_transport. 1937d6e971d8SJeff Layton */ 1938a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 1939a564b8f0SMel Gorman { 1940a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1941a564b8f0SMel Gorman xprt); 1942a564b8f0SMel Gorman 1943d6e971d8SJeff Layton /* 1944d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The 1945d6e971d8SJeff Layton * reconnecting process will get it for us. 1946d6e971d8SJeff Layton */ 1947d6e971d8SJeff Layton if (!transport->inet) 1948d6e971d8SJeff Layton return; 19498e228133SJeff Layton if (atomic_read(&xprt->swapper)) 1950a564b8f0SMel Gorman sk_set_memalloc(transport->inet); 1951a564b8f0SMel Gorman } 1952a564b8f0SMel Gorman 1953a564b8f0SMel Gorman /** 1954d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap. 1955a564b8f0SMel Gorman * @xprt: transport to tag 1956a564b8f0SMel Gorman * 19578e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and 19588e228133SJeff Layton * optionally mark it for swapping if it wasn't already. 1959a564b8f0SMel Gorman */ 1960d67fa4d8SJeff Layton static int 1961d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 1962a564b8f0SMel Gorman { 1963d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 1964a564b8f0SMel Gorman 1965d6e971d8SJeff Layton if (atomic_inc_return(&xprt->swapper) != 1) 1966d6e971d8SJeff Layton return 0; 1967d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 1968d6e971d8SJeff Layton return -ERESTARTSYS; 1969d6e971d8SJeff Layton if (xs->inet) 1970d6e971d8SJeff Layton sk_set_memalloc(xs->inet); 1971d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 19728e228133SJeff Layton return 0; 1973a564b8f0SMel Gorman } 1974a564b8f0SMel Gorman 19758e228133SJeff Layton /** 1976d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap. 19778e228133SJeff Layton * @xprt: transport to tag 19788e228133SJeff Layton * 19798e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the 19808e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket. 19818e228133SJeff Layton */ 1982d67fa4d8SJeff Layton static void 1983d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 19848e228133SJeff Layton { 1985d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 19868e228133SJeff Layton 1987d6e971d8SJeff Layton if (!atomic_dec_and_test(&xprt->swapper)) 1988d6e971d8SJeff Layton return; 1989d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 1990d6e971d8SJeff Layton return; 1991d6e971d8SJeff Layton if (xs->inet) 1992d6e971d8SJeff Layton sk_clear_memalloc(xs->inet); 1993d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 1994a564b8f0SMel Gorman } 1995a564b8f0SMel Gorman #else 1996a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 1997a564b8f0SMel Gorman { 1998a564b8f0SMel Gorman } 1999d67fa4d8SJeff Layton 2000d67fa4d8SJeff Layton static int 2001d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2002d67fa4d8SJeff Layton { 2003d67fa4d8SJeff Layton return -EINVAL; 2004d67fa4d8SJeff Layton } 2005d67fa4d8SJeff Layton 2006d67fa4d8SJeff Layton static void 2007d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 2008d67fa4d8SJeff Layton { 2009d67fa4d8SJeff Layton } 2010a564b8f0SMel Gorman #endif 2011a564b8f0SMel Gorman 201216be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2013a246b010SChuck Lever { 201416be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2015edb267a6SChuck Lever 2016ee0ac0c2SChuck Lever if (!transport->inet) { 2017b0d93ad5SChuck Lever struct sock *sk = sock->sk; 2018b0d93ad5SChuck Lever 2019b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2020b0d93ad5SChuck Lever 20212a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 20222a9e1cfaSTrond Myklebust 2023b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2024f9b2ee71STrond Myklebust sk->sk_data_ready = xs_data_ready; 2025b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space; 2026b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 2027b0d93ad5SChuck Lever 2028b0d93ad5SChuck Lever xprt_set_connected(xprt); 2029b0d93ad5SChuck Lever 2030b0d93ad5SChuck Lever /* Reset to new socket */ 2031ee0ac0c2SChuck Lever transport->sock = sock; 2032ee0ac0c2SChuck Lever transport->inet = sk; 2033b0d93ad5SChuck Lever 2034a564b8f0SMel Gorman xs_set_memalloc(xprt); 2035a564b8f0SMel Gorman 2036b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2037b0d93ad5SChuck Lever } 2038470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 203902910177STrond Myklebust 204002910177STrond Myklebust xprt->stat.connect_start = jiffies; 204116be2d20SChuck Lever } 204216be2d20SChuck Lever 20438c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work) 2044a246b010SChuck Lever { 2045a246b010SChuck Lever struct sock_xprt *transport = 2046a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work); 2047a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2048d099b8afSColin Ian King struct socket *sock; 2049b65c0310SPavel Emelyanov int status = -EIO; 2050a246b010SChuck Lever 20518c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport, 20524dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM, 20534dda9c8aSTrond Myklebust IPPROTO_UDP, false); 2054b65c0310SPavel Emelyanov if (IS_ERR(sock)) 2055a246b010SChuck Lever goto out; 205668e220bdSChuck Lever 2057c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2058c740eff8SChuck Lever "%s (port %s)\n", xprt, 2059c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2060c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2061c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 206268e220bdSChuck Lever 206368e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock); 206440b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0); 2065a246b010SChuck Lever status = 0; 2066b0d93ad5SChuck Lever out: 2067b0d93ad5SChuck Lever xprt_clear_connecting(xprt); 2068cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport); 20697d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2070b0d93ad5SChuck Lever } 2071b0d93ad5SChuck Lever 20724876cc77STrond Myklebust /** 20734876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket 20744876cc77STrond Myklebust * @xprt: transport 20754876cc77STrond Myklebust * 20764876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the 20774876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR); 20784876cc77STrond Myklebust */ 20794876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt) 20804876cc77STrond Myklebust { 20814876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 20824876cc77STrond Myklebust struct socket *sock = transport->sock; 20839b30889cSTrond Myklebust int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE; 20844876cc77STrond Myklebust 20854876cc77STrond Myklebust if (sock == NULL) 20864876cc77STrond Myklebust return; 20879b30889cSTrond Myklebust switch (skst) { 20889b30889cSTrond Myklebust default: 20894876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 20904876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock); 20919b30889cSTrond Myklebust break; 20929b30889cSTrond Myklebust case TCP_CLOSE: 20939b30889cSTrond Myklebust case TCP_TIME_WAIT: 20944876cc77STrond Myklebust xs_reset_transport(transport); 20954876cc77STrond Myklebust } 20969b30889cSTrond Myklebust } 20974876cc77STrond Myklebust 20988d1b8c62STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 20998d1b8c62STrond Myklebust struct socket *sock) 2100b0d93ad5SChuck Lever { 210116be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21027196dbb0STrond Myklebust unsigned int keepidle; 21037196dbb0STrond Myklebust unsigned int keepcnt; 2104775f06abSTrond Myklebust unsigned int timeo; 21057f260e85STrond Myklebust 2106b5e92419STrond Myklebust spin_lock(&xprt->transport_lock); 21077196dbb0STrond Myklebust keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ); 21087196dbb0STrond Myklebust keepcnt = xprt->timeout->to_retries + 1; 21097196dbb0STrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) * 21107196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 21117196dbb0STrond Myklebust clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 2112b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock); 21137f260e85STrond Myklebust 21147f260e85STrond Myklebust /* TCP Keepalive options */ 2115ce3d9544SChristoph Hellwig sock_set_keepalive(sock->sk); 211671c48eb8SChristoph Hellwig tcp_sock_set_keepidle(sock->sk, keepidle); 2117d41ecaacSChristoph Hellwig tcp_sock_set_keepintvl(sock->sk, keepidle); 2118480aeb96SChristoph Hellwig tcp_sock_set_keepcnt(sock->sk, keepcnt); 2119b0d93ad5SChuck Lever 21208d1b8c62STrond Myklebust /* TCP user timeout (see RFC5482) */ 2121c488aeadSChristoph Hellwig tcp_sock_set_user_timeout(sock->sk, timeo); 21228d1b8c62STrond Myklebust } 21238d1b8c62STrond Myklebust 21247196dbb0STrond Myklebust static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt, 21257196dbb0STrond Myklebust unsigned long connect_timeout, 21267196dbb0STrond Myklebust unsigned long reconnect_timeout) 21277196dbb0STrond Myklebust { 21287196dbb0STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21297196dbb0STrond Myklebust struct rpc_timeout to; 21307196dbb0STrond Myklebust unsigned long initval; 21317196dbb0STrond Myklebust 2132b5e92419STrond Myklebust spin_lock(&xprt->transport_lock); 21337196dbb0STrond Myklebust if (reconnect_timeout < xprt->max_reconnect_timeout) 21347196dbb0STrond Myklebust xprt->max_reconnect_timeout = reconnect_timeout; 21357196dbb0STrond Myklebust if (connect_timeout < xprt->connect_timeout) { 21367196dbb0STrond Myklebust memcpy(&to, xprt->timeout, sizeof(to)); 21377196dbb0STrond Myklebust initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1); 21387196dbb0STrond Myklebust /* Arbitrary lower limit */ 21397196dbb0STrond Myklebust if (initval < XS_TCP_INIT_REEST_TO << 1) 21407196dbb0STrond Myklebust initval = XS_TCP_INIT_REEST_TO << 1; 21417196dbb0STrond Myklebust to.to_initval = initval; 21427196dbb0STrond Myklebust to.to_maxval = initval; 21437196dbb0STrond Myklebust memcpy(&transport->tcp_timeout, &to, 21447196dbb0STrond Myklebust sizeof(transport->tcp_timeout)); 21457196dbb0STrond Myklebust xprt->timeout = &transport->tcp_timeout; 21467196dbb0STrond Myklebust xprt->connect_timeout = connect_timeout; 21477196dbb0STrond Myklebust } 21487196dbb0STrond Myklebust set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 2149b5e92419STrond Myklebust spin_unlock(&xprt->transport_lock); 21507196dbb0STrond Myklebust } 21517196dbb0STrond Myklebust 21528d1b8c62STrond Myklebust static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 21538d1b8c62STrond Myklebust { 21548d1b8c62STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21558d1b8c62STrond Myklebust int ret = -ENOTCONN; 21568d1b8c62STrond Myklebust 21578d1b8c62STrond Myklebust if (!transport->inet) { 21588d1b8c62STrond Myklebust struct sock *sk = sock->sk; 21598d1b8c62STrond Myklebust 2160d88e4d82SNeilBrown /* Avoid temporary address, they are bad for long-lived 2161d88e4d82SNeilBrown * connections such as NFS mounts. 2162d88e4d82SNeilBrown * RFC4941, section 3.6 suggests that: 2163d88e4d82SNeilBrown * Individual applications, which have specific 2164d88e4d82SNeilBrown * knowledge about the normal duration of connections, 2165d88e4d82SNeilBrown * MAY override this as appropriate. 2166d88e4d82SNeilBrown */ 216718d5ad62SChristoph Hellwig if (xs_addr(xprt)->sa_family == PF_INET6) { 216818d5ad62SChristoph Hellwig ip6_sock_set_addr_preferences(sk, 216918d5ad62SChristoph Hellwig IPV6_PREFER_SRC_PUBLIC); 217018d5ad62SChristoph Hellwig } 2171d88e4d82SNeilBrown 21728d1b8c62STrond Myklebust xs_tcp_set_socket_timeouts(xprt, sock); 2173d737e5d4STrond Myklebust tcp_sock_set_nodelay(sk); 2174775f06abSTrond Myklebust 2175b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2176b0d93ad5SChuck Lever 21772a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 21782a9e1cfaSTrond Myklebust 2179b0d93ad5SChuck Lever sk->sk_user_data = xprt; 21805157b956STrond Myklebust sk->sk_data_ready = xs_data_ready; 2181b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change; 2182b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space; 2183b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 21842118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 21853167e12cSChuck Lever 21863167e12cSChuck Lever /* socket options */ 21873167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER); 2188b0d93ad5SChuck Lever 2189b0d93ad5SChuck Lever xprt_clear_connected(xprt); 2190b0d93ad5SChuck Lever 2191b0d93ad5SChuck Lever /* Reset to new socket */ 2192ee0ac0c2SChuck Lever transport->sock = sock; 2193ee0ac0c2SChuck Lever transport->inet = sk; 2194b0d93ad5SChuck Lever 2195b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2196b0d93ad5SChuck Lever } 2197b0d93ad5SChuck Lever 219801d37c42STrond Myklebust if (!xprt_bound(xprt)) 2199fe19a96bSTrond Myklebust goto out; 220001d37c42STrond Myklebust 2201a564b8f0SMel Gorman xs_set_memalloc(xprt); 2202a564b8f0SMel Gorman 2203ae053551STrond Myklebust xs_stream_start_connect(transport); 2204e1806c7bSTrond Myklebust 2205b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */ 22060fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 2207fe19a96bSTrond Myklebust ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK); 2208fe19a96bSTrond Myklebust switch (ret) { 2209fe19a96bSTrond Myklebust case 0: 22104dda9c8aSTrond Myklebust xs_set_srcport(transport, sock); 2211df561f66SGustavo A. R. Silva fallthrough; 2212fe19a96bSTrond Myklebust case -EINPROGRESS: 2213fe19a96bSTrond Myklebust /* SYN_SENT! */ 2214fe19a96bSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 2215fe19a96bSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 22161f4c17a0STrond Myklebust break; 22171f4c17a0STrond Myklebust case -EADDRNOTAVAIL: 22181f4c17a0STrond Myklebust /* Source port number is unavailable. Try a new one! */ 22191f4c17a0STrond Myklebust transport->srcport = 0; 2220fe19a96bSTrond Myklebust } 2221fe19a96bSTrond Myklebust out: 2222fe19a96bSTrond Myklebust return ret; 222316be2d20SChuck Lever } 222416be2d20SChuck Lever 222516be2d20SChuck Lever /** 2226b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint 2227acf0a39fSChuck Lever * @work: queued work item 222816be2d20SChuck Lever * 222916be2d20SChuck Lever * Invoked by a work queue tasklet. 223016be2d20SChuck Lever */ 2231cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work) 223216be2d20SChuck Lever { 2233cdd518d5SPavel Emelyanov struct sock_xprt *transport = 2234cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work); 223516be2d20SChuck Lever struct socket *sock = transport->sock; 2236a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt; 2237b61d59ffSTrond Myklebust int status = -EIO; 223816be2d20SChuck Lever 223916be2d20SChuck Lever if (!sock) { 2240cdd518d5SPavel Emelyanov sock = xs_create_sock(xprt, transport, 22414dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_STREAM, 22424dda9c8aSTrond Myklebust IPPROTO_TCP, true); 2243b61d59ffSTrond Myklebust if (IS_ERR(sock)) { 2244b61d59ffSTrond Myklebust status = PTR_ERR(sock); 224516be2d20SChuck Lever goto out; 224616be2d20SChuck Lever } 22477d1e8255STrond Myklebust } 22487d1e8255STrond Myklebust 2249c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2250c740eff8SChuck Lever "%s (port %s)\n", xprt, 2251c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2252c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2253c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 225416be2d20SChuck Lever 225516be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock); 225640b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2257a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n", 225846121cf7SChuck Lever xprt, -status, xprt_connected(xprt), 225946121cf7SChuck Lever sock->sk->sk_state); 2260a246b010SChuck Lever switch (status) { 2261f75e6745STrond Myklebust default: 2262f75e6745STrond Myklebust printk("%s: connect returned unhandled error %d\n", 2263f75e6745STrond Myklebust __func__, status); 2264df561f66SGustavo A. R. Silva fallthrough; 2265f75e6745STrond Myklebust case -EADDRNOTAVAIL: 2266f75e6745STrond Myklebust /* We're probably in TIME_WAIT. Get rid of existing socket, 2267f75e6745STrond Myklebust * and retry 2268f75e6745STrond Myklebust */ 2269a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 227088b5ed73STrond Myklebust break; 22712a491991STrond Myklebust case 0: 2272a246b010SChuck Lever case -EINPROGRESS: 2273a246b010SChuck Lever case -EALREADY: 2274718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 22757d1e8255STrond Myklebust return; 22769fcfe0c8STrond Myklebust case -EINVAL: 22779fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link 22789fcfe0c8STrond Myklebust * local IPv6 address without a scope-id. 22799fcfe0c8STrond Myklebust */ 22803ed5e2a2STrond Myklebust case -ECONNREFUSED: 22813ed5e2a2STrond Myklebust case -ECONNRESET: 2282eb5b46faSTrond Myklebust case -ENETDOWN: 22833ed5e2a2STrond Myklebust case -ENETUNREACH: 22844ba161a7STrond Myklebust case -EHOSTUNREACH: 22853913c78cSTrond Myklebust case -EADDRINUSE: 22863601c4a9STrond Myklebust case -ENOBUFS: 22878c71139dSCalum Mackay /* xs_tcp_force_close() wakes tasks with a fixed error code. 22888c71139dSCalum Mackay * We need to wake them first to ensure the correct error code. 22896ea44adcSNeilBrown */ 22906ea44adcSNeilBrown xprt_wake_pending_tasks(xprt, status); 22914efdd92cSTrond Myklebust xs_tcp_force_close(xprt); 22929fcfe0c8STrond Myklebust goto out; 22938a2cec29STrond Myklebust } 22942a491991STrond Myklebust status = -EAGAIN; 2295a246b010SChuck Lever out: 22962226feb6SChuck Lever xprt_clear_connecting(xprt); 2297cf76785dSTrond Myklebust xprt_unlock_connect(xprt, transport); 22987d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2299a246b010SChuck Lever } 2300a246b010SChuck Lever 230168e220bdSChuck Lever /** 23029903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint 23031b092092STrond Myklebust * @xprt: pointer to transport structure 23049903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request 23059903cd1cSChuck Lever * 23069903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting. 230703bf4b70SChuck Lever * 230803bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway 230903bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a 231003bf4b70SChuck Lever * socket on a privileged port. 231103bf4b70SChuck Lever * 231203bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents 231303bf4b70SChuck Lever * retry floods (hard mounts). 23149903cd1cSChuck Lever */ 23151b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2316a246b010SChuck Lever { 2317ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 231802910177STrond Myklebust unsigned long delay = 0; 2319a246b010SChuck Lever 2320718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport)); 2321718ba5b8STrond Myklebust 232299b1a4c3STrond Myklebust if (transport->sock != NULL) { 232346121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu " 232446121cf7SChuck Lever "seconds\n", 232503bf4b70SChuck Lever xprt, xprt->reestablish_timeout / HZ); 232699b1a4c3STrond Myklebust 232799b1a4c3STrond Myklebust /* Start by resetting any existing state */ 232899b1a4c3STrond Myklebust xs_reset_transport(transport); 232999b1a4c3STrond Myklebust 2330675dd90aSChuck Lever delay = xprt_reconnect_delay(xprt); 2331675dd90aSChuck Lever xprt_reconnect_backoff(xprt, XS_TCP_INIT_REEST_TO); 233202910177STrond Myklebust 233302910177STrond Myklebust } else 23349903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 233502910177STrond Myklebust 233640a5f1b1STrond Myklebust queue_delayed_work(xprtiod_workqueue, 233702910177STrond Myklebust &transport->connect_worker, 233802910177STrond Myklebust delay); 2339a246b010SChuck Lever } 2340a246b010SChuck Lever 23414f8943f8STrond Myklebust static void xs_wake_disconnect(struct sock_xprt *transport) 23424f8943f8STrond Myklebust { 23434f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_DISCONNECT, &transport->sock_state)) 23444f8943f8STrond Myklebust xs_tcp_force_close(&transport->xprt); 23454f8943f8STrond Myklebust } 23464f8943f8STrond Myklebust 23474f8943f8STrond Myklebust static void xs_wake_write(struct sock_xprt *transport) 23484f8943f8STrond Myklebust { 23494f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_WRITE, &transport->sock_state)) 23504f8943f8STrond Myklebust xprt_write_space(&transport->xprt); 23514f8943f8STrond Myklebust } 23524f8943f8STrond Myklebust 23534f8943f8STrond Myklebust static void xs_wake_error(struct sock_xprt *transport) 23544f8943f8STrond Myklebust { 23554f8943f8STrond Myklebust int sockerr; 23564f8943f8STrond Myklebust 23574f8943f8STrond Myklebust if (!test_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state)) 23584f8943f8STrond Myklebust return; 23594f8943f8STrond Myklebust mutex_lock(&transport->recv_mutex); 23604f8943f8STrond Myklebust if (transport->sock == NULL) 23614f8943f8STrond Myklebust goto out; 23624f8943f8STrond Myklebust if (!test_and_clear_bit(XPRT_SOCK_WAKE_ERROR, &transport->sock_state)) 23634f8943f8STrond Myklebust goto out; 2364af84537dSBenjamin Coddington sockerr = xchg(&transport->xprt_err, 0); 23654f8943f8STrond Myklebust if (sockerr < 0) 23664f8943f8STrond Myklebust xprt_wake_pending_tasks(&transport->xprt, sockerr); 23674f8943f8STrond Myklebust out: 23684f8943f8STrond Myklebust mutex_unlock(&transport->recv_mutex); 23694f8943f8STrond Myklebust } 23704f8943f8STrond Myklebust 23714f8943f8STrond Myklebust static void xs_wake_pending(struct sock_xprt *transport) 23724f8943f8STrond Myklebust { 23734f8943f8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_WAKE_PENDING, &transport->sock_state)) 23744f8943f8STrond Myklebust xprt_wake_pending_tasks(&transport->xprt, -EAGAIN); 23754f8943f8STrond Myklebust } 23764f8943f8STrond Myklebust 23774f8943f8STrond Myklebust static void xs_error_handle(struct work_struct *work) 23784f8943f8STrond Myklebust { 23794f8943f8STrond Myklebust struct sock_xprt *transport = container_of(work, 23804f8943f8STrond Myklebust struct sock_xprt, error_worker); 23814f8943f8STrond Myklebust 23824f8943f8STrond Myklebust xs_wake_disconnect(transport); 23834f8943f8STrond Myklebust xs_wake_write(transport); 23844f8943f8STrond Myklebust xs_wake_error(transport); 23854f8943f8STrond Myklebust xs_wake_pending(transport); 23864f8943f8STrond Myklebust } 23874f8943f8STrond Myklebust 2388262ca07dSChuck Lever /** 238912b20ce3SBhaskar Chowdhury * xs_local_print_stats - display AF_LOCAL socket-specific stats 2390176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics 2391176e21eeSChuck Lever * @seq: output file 2392176e21eeSChuck Lever * 2393176e21eeSChuck Lever */ 2394176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2395176e21eeSChuck Lever { 2396176e21eeSChuck Lever long idle_time = 0; 2397176e21eeSChuck Lever 2398176e21eeSChuck Lever if (xprt_connected(xprt)) 2399176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2400176e21eeSChuck Lever 2401176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu " 240215a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2403176e21eeSChuck Lever xprt->stat.bind_count, 2404176e21eeSChuck Lever xprt->stat.connect_count, 24058440a886SChuck Lever xprt->stat.connect_time / HZ, 2406176e21eeSChuck Lever idle_time, 2407176e21eeSChuck Lever xprt->stat.sends, 2408176e21eeSChuck Lever xprt->stat.recvs, 2409176e21eeSChuck Lever xprt->stat.bad_xids, 2410176e21eeSChuck Lever xprt->stat.req_u, 241115a45206SAndy Adamson xprt->stat.bklog_u, 241215a45206SAndy Adamson xprt->stat.max_slots, 241315a45206SAndy Adamson xprt->stat.sending_u, 241415a45206SAndy Adamson xprt->stat.pending_u); 2415176e21eeSChuck Lever } 2416176e21eeSChuck Lever 2417176e21eeSChuck Lever /** 241812b20ce3SBhaskar Chowdhury * xs_udp_print_stats - display UDP socket-specific stats 2419262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2420262ca07dSChuck Lever * @seq: output file 2421262ca07dSChuck Lever * 2422262ca07dSChuck Lever */ 2423262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2424262ca07dSChuck Lever { 2425c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2426c8475461SChuck Lever 242715a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu " 242815a45206SAndy Adamson "%lu %llu %llu\n", 2429fbfffbd5SChuck Lever transport->srcport, 2430262ca07dSChuck Lever xprt->stat.bind_count, 2431262ca07dSChuck Lever xprt->stat.sends, 2432262ca07dSChuck Lever xprt->stat.recvs, 2433262ca07dSChuck Lever xprt->stat.bad_xids, 2434262ca07dSChuck Lever xprt->stat.req_u, 243515a45206SAndy Adamson xprt->stat.bklog_u, 243615a45206SAndy Adamson xprt->stat.max_slots, 243715a45206SAndy Adamson xprt->stat.sending_u, 243815a45206SAndy Adamson xprt->stat.pending_u); 2439262ca07dSChuck Lever } 2440262ca07dSChuck Lever 2441262ca07dSChuck Lever /** 244212b20ce3SBhaskar Chowdhury * xs_tcp_print_stats - display TCP socket-specific stats 2443262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2444262ca07dSChuck Lever * @seq: output file 2445262ca07dSChuck Lever * 2446262ca07dSChuck Lever */ 2447262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2448262ca07dSChuck Lever { 2449c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2450262ca07dSChuck Lever long idle_time = 0; 2451262ca07dSChuck Lever 2452262ca07dSChuck Lever if (xprt_connected(xprt)) 2453262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2454262ca07dSChuck Lever 245515a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu " 245615a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2457fbfffbd5SChuck Lever transport->srcport, 2458262ca07dSChuck Lever xprt->stat.bind_count, 2459262ca07dSChuck Lever xprt->stat.connect_count, 24608440a886SChuck Lever xprt->stat.connect_time / HZ, 2461262ca07dSChuck Lever idle_time, 2462262ca07dSChuck Lever xprt->stat.sends, 2463262ca07dSChuck Lever xprt->stat.recvs, 2464262ca07dSChuck Lever xprt->stat.bad_xids, 2465262ca07dSChuck Lever xprt->stat.req_u, 246615a45206SAndy Adamson xprt->stat.bklog_u, 246715a45206SAndy Adamson xprt->stat.max_slots, 246815a45206SAndy Adamson xprt->stat.sending_u, 246915a45206SAndy Adamson xprt->stat.pending_u); 2470262ca07dSChuck Lever } 2471262ca07dSChuck Lever 24724cfc7e60SRahul Iyer /* 24734cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason 24744cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want 24754cfc7e60SRahul Iyer * to use the server side send routines. 24764cfc7e60SRahul Iyer */ 24775fe6eaa1SChuck Lever static int bc_malloc(struct rpc_task *task) 24784cfc7e60SRahul Iyer { 24795fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp; 24805fe6eaa1SChuck Lever size_t size = rqst->rq_callsize; 24814cfc7e60SRahul Iyer struct page *page; 24824cfc7e60SRahul Iyer struct rpc_buffer *buf; 24834cfc7e60SRahul Iyer 24845fe6eaa1SChuck Lever if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) { 24855fe6eaa1SChuck Lever WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n", 24865fe6eaa1SChuck Lever size); 24875fe6eaa1SChuck Lever return -EINVAL; 24885fe6eaa1SChuck Lever } 24894cfc7e60SRahul Iyer 2490b8a13d03SWeston Andros Adamson page = alloc_page(GFP_KERNEL); 24914cfc7e60SRahul Iyer if (!page) 24925fe6eaa1SChuck Lever return -ENOMEM; 24934cfc7e60SRahul Iyer 24944cfc7e60SRahul Iyer buf = page_address(page); 24954cfc7e60SRahul Iyer buf->len = PAGE_SIZE; 24964cfc7e60SRahul Iyer 24975fe6eaa1SChuck Lever rqst->rq_buffer = buf->data; 249818e601d6SJeff Layton rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; 24995fe6eaa1SChuck Lever return 0; 25004cfc7e60SRahul Iyer } 25014cfc7e60SRahul Iyer 25024cfc7e60SRahul Iyer /* 25034cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine 25044cfc7e60SRahul Iyer */ 25053435c74aSChuck Lever static void bc_free(struct rpc_task *task) 25064cfc7e60SRahul Iyer { 25073435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer; 25084cfc7e60SRahul Iyer struct rpc_buffer *buf; 25094cfc7e60SRahul Iyer 25104cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data); 25114cfc7e60SRahul Iyer free_page((unsigned long)buf); 25124cfc7e60SRahul Iyer } 25134cfc7e60SRahul Iyer 25144cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req) 25154cfc7e60SRahul Iyer { 2516da1661b9SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 25174cfc7e60SRahul Iyer struct sock_xprt *transport = 2518067fb11bSChuck Lever container_of(req->rq_xprt, struct sock_xprt, xprt); 2519067fb11bSChuck Lever struct msghdr msg = { 2520da1661b9SChuck Lever .msg_flags = 0, 2521067fb11bSChuck Lever }; 2522067fb11bSChuck Lever rpc_fraghdr marker = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | 2523da1661b9SChuck Lever (u32)xdr->len); 2524da1661b9SChuck Lever unsigned int sent = 0; 2525da1661b9SChuck Lever int err; 25264cfc7e60SRahul Iyer 25278729aabaSChuck Lever req->rq_xtime = ktime_get(); 2528da1661b9SChuck Lever err = xprt_sock_sendmsg(transport->sock, &msg, xdr, 0, marker, &sent); 2529da1661b9SChuck Lever xdr_free_bvec(xdr); 2530da1661b9SChuck Lever if (err < 0 || sent != (xdr->len + sizeof(marker))) 2531067fb11bSChuck Lever return -EAGAIN; 2532da1661b9SChuck Lever return sent; 25334cfc7e60SRahul Iyer } 25344cfc7e60SRahul Iyer 2535ca4faf54SChuck Lever /** 2536ca4faf54SChuck Lever * bc_send_request - Send a backchannel Call on a TCP socket 2537ca4faf54SChuck Lever * @req: rpc_rqst containing Call message to be sent 2538ca4faf54SChuck Lever * 2539ca4faf54SChuck Lever * xpt_mutex ensures @rqstp's whole message is written to the socket 2540ca4faf54SChuck Lever * without interruption. 2541ca4faf54SChuck Lever * 2542ca4faf54SChuck Lever * Return values: 2543ca4faf54SChuck Lever * %0 if the message was sent successfully 2544ca4faf54SChuck Lever * %ENOTCONN if the message was not sent 25454cfc7e60SRahul Iyer */ 2546adfa7144STrond Myklebust static int bc_send_request(struct rpc_rqst *req) 25474cfc7e60SRahul Iyer { 25484cfc7e60SRahul Iyer struct svc_xprt *xprt; 25497fc56136SAndrzej Hajda int len; 25504cfc7e60SRahul Iyer 25514cfc7e60SRahul Iyer /* 25524cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt 25534cfc7e60SRahul Iyer */ 25544cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt; 25554cfc7e60SRahul Iyer 25564cfc7e60SRahul Iyer /* 25574cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared 25584cfc7e60SRahul Iyer * with the fore channel 25594cfc7e60SRahul Iyer */ 2560c544577dSTrond Myklebust mutex_lock(&xprt->xpt_mutex); 25614cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags)) 25624cfc7e60SRahul Iyer len = -ENOTCONN; 25634cfc7e60SRahul Iyer else 25644cfc7e60SRahul Iyer len = bc_sendto(req); 25654cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex); 25664cfc7e60SRahul Iyer 25674cfc7e60SRahul Iyer if (len > 0) 25684cfc7e60SRahul Iyer len = 0; 25694cfc7e60SRahul Iyer 25704cfc7e60SRahul Iyer return len; 25714cfc7e60SRahul Iyer } 25724cfc7e60SRahul Iyer 25734cfc7e60SRahul Iyer /* 25744cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing 25754cfc7e60SRahul Iyer */ 25764cfc7e60SRahul Iyer 25774cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt) 25784cfc7e60SRahul Iyer { 25796221f1d9SChuck Lever xprt_disconnect_done(xprt); 25804cfc7e60SRahul Iyer } 25814cfc7e60SRahul Iyer 25824cfc7e60SRahul Iyer /* 25834cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client 25844cfc7e60SRahul Iyer * initiated, we do nothing 25854cfc7e60SRahul Iyer */ 25864cfc7e60SRahul Iyer 25874cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt) 25884cfc7e60SRahul Iyer { 258947f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt); 259047f72efaSKinglong Mee 259147f72efaSKinglong Mee xs_xprt_free(xprt); 259247f72efaSKinglong Mee module_put(THIS_MODULE); 25934cfc7e60SRahul Iyer } 25944cfc7e60SRahul Iyer 2595d31ae254SChuck Lever static const struct rpc_xprt_ops xs_local_ops = { 2596176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt, 25974cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 2598f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2599a9cde23aSChuck Lever .free_slot = xprt_free_slot, 2600176e21eeSChuck Lever .rpcbind = xs_local_rpcbind, 2601176e21eeSChuck Lever .set_port = xs_local_set_port, 2602dc107402SJ. Bruce Fields .connect = xs_local_connect, 2603176e21eeSChuck Lever .buf_alloc = rpc_malloc, 2604176e21eeSChuck Lever .buf_free = rpc_free, 2605550aebfeSTrond Myklebust .prepare_request = xs_stream_prepare_request, 2606176e21eeSChuck Lever .send_request = xs_local_send_request, 26078ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def, 2608176e21eeSChuck Lever .close = xs_close, 2609a1311d87STrond Myklebust .destroy = xs_destroy, 2610176e21eeSChuck Lever .print_stats = xs_local_print_stats, 2611d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2612d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 2613176e21eeSChuck Lever }; 2614176e21eeSChuck Lever 2615d31ae254SChuck Lever static const struct rpc_xprt_ops xs_udp_ops = { 261643118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size, 261712a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong, 261849e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong, 2619f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2620a9cde23aSChuck Lever .free_slot = xprt_free_slot, 262145160d62SChuck Lever .rpcbind = rpcb_getport_async, 262292200412SChuck Lever .set_port = xs_set_port, 26239903cd1cSChuck Lever .connect = xs_connect, 262402107148SChuck Lever .buf_alloc = rpc_malloc, 262502107148SChuck Lever .buf_free = rpc_free, 2626262965f5SChuck Lever .send_request = xs_udp_send_request, 26278ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_rtt, 262846c0ee8bSChuck Lever .timer = xs_udp_timer, 2629a58dd398SChuck Lever .release_request = xprt_release_rqst_cong, 2630262965f5SChuck Lever .close = xs_close, 2631262965f5SChuck Lever .destroy = xs_destroy, 2632262ca07dSChuck Lever .print_stats = xs_udp_print_stats, 2633d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2634d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26354a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2636262965f5SChuck Lever }; 2637262965f5SChuck Lever 2638d31ae254SChuck Lever static const struct rpc_xprt_ops xs_tcp_ops = { 263912a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt, 26404cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 264136bd7de9STrond Myklebust .alloc_slot = xprt_alloc_slot, 2642a9cde23aSChuck Lever .free_slot = xprt_free_slot, 264345160d62SChuck Lever .rpcbind = rpcb_getport_async, 264492200412SChuck Lever .set_port = xs_set_port, 26450b9e7943STrond Myklebust .connect = xs_connect, 264602107148SChuck Lever .buf_alloc = rpc_malloc, 264702107148SChuck Lever .buf_free = rpc_free, 2648277e4ab7STrond Myklebust .prepare_request = xs_stream_prepare_request, 2649262965f5SChuck Lever .send_request = xs_tcp_send_request, 26508ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def, 2651c627d31bSTrond Myklebust .close = xs_tcp_shutdown, 26529903cd1cSChuck Lever .destroy = xs_destroy, 26537196dbb0STrond Myklebust .set_connect_timeout = xs_tcp_set_connect_timeout, 2654262ca07dSChuck Lever .print_stats = xs_tcp_print_stats, 2655d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2656d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26574a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 265842e5c3e2SChuck Lever #ifdef CONFIG_SUNRPC_BACKCHANNEL 265942e5c3e2SChuck Lever .bc_setup = xprt_setup_bc, 26606b26cc8cSChuck Lever .bc_maxpayload = xs_tcp_bc_maxpayload, 26617402a4feSTrond Myklebust .bc_num_slots = xprt_bc_max_slots, 266242e5c3e2SChuck Lever .bc_free_rqst = xprt_free_bc_rqst, 266342e5c3e2SChuck Lever .bc_destroy = xprt_destroy_bc, 266442e5c3e2SChuck Lever #endif 2665a246b010SChuck Lever }; 2666a246b010SChuck Lever 26674cfc7e60SRahul Iyer /* 26684cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel 26694cfc7e60SRahul Iyer */ 26704cfc7e60SRahul Iyer 2671d31ae254SChuck Lever static const struct rpc_xprt_ops bc_tcp_ops = { 26724cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt, 26734cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt, 267484e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot, 2675a9cde23aSChuck Lever .free_slot = xprt_free_slot, 26764cfc7e60SRahul Iyer .buf_alloc = bc_malloc, 26774cfc7e60SRahul Iyer .buf_free = bc_free, 26784cfc7e60SRahul Iyer .send_request = bc_send_request, 26798ba6a92dSTrond Myklebust .wait_for_reply_request = xprt_wait_for_reply_request_def, 26804cfc7e60SRahul Iyer .close = bc_close, 26814cfc7e60SRahul Iyer .destroy = bc_destroy, 26824cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats, 2683d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2684d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26854a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 26864cfc7e60SRahul Iyer }; 26874cfc7e60SRahul Iyer 268892476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap) 268992476850SChuck Lever { 269092476850SChuck Lever static const struct sockaddr_in sin = { 269192476850SChuck Lever .sin_family = AF_INET, 269292476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY), 269392476850SChuck Lever }; 269492476850SChuck Lever static const struct sockaddr_in6 sin6 = { 269592476850SChuck Lever .sin6_family = AF_INET6, 269692476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT, 269792476850SChuck Lever }; 269892476850SChuck Lever 269992476850SChuck Lever switch (family) { 2700176e21eeSChuck Lever case AF_LOCAL: 2701176e21eeSChuck Lever break; 270292476850SChuck Lever case AF_INET: 270392476850SChuck Lever memcpy(sap, &sin, sizeof(sin)); 270492476850SChuck Lever break; 270592476850SChuck Lever case AF_INET6: 270692476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6)); 270792476850SChuck Lever break; 270892476850SChuck Lever default: 270992476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__); 271092476850SChuck Lever return -EAFNOSUPPORT; 271192476850SChuck Lever } 271292476850SChuck Lever return 0; 271392476850SChuck Lever } 271492476850SChuck Lever 27153c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args, 2716d9ba131dSTrond Myklebust unsigned int slot_table_size, 2717d9ba131dSTrond Myklebust unsigned int max_slot_table_size) 2718c8541ecdSChuck Lever { 2719c8541ecdSChuck Lever struct rpc_xprt *xprt; 2720ffc2e518SChuck Lever struct sock_xprt *new; 2721c8541ecdSChuck Lever 272296802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) { 2723c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n"); 2724c8541ecdSChuck Lever return ERR_PTR(-EBADF); 2725c8541ecdSChuck Lever } 2726c8541ecdSChuck Lever 2727d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size, 2728d9ba131dSTrond Myklebust max_slot_table_size); 2729bd1722d4SPavel Emelyanov if (xprt == NULL) { 273046121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate " 273146121cf7SChuck Lever "rpc_xprt\n"); 2732c8541ecdSChuck Lever return ERR_PTR(-ENOMEM); 2733c8541ecdSChuck Lever } 2734c8541ecdSChuck Lever 2735bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt); 2736edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex); 273796802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen); 273896802a09SFrank van Maarseveen xprt->addrlen = args->addrlen; 2739d3bc9a1dSFrank van Maarseveen if (args->srcaddr) 2740fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen); 274192476850SChuck Lever else { 274292476850SChuck Lever int err; 274392476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family, 274492476850SChuck Lever (struct sockaddr *)&new->srcaddr); 27452aa13531SStanislav Kinsbursky if (err != 0) { 27462aa13531SStanislav Kinsbursky xprt_free(xprt); 274792476850SChuck Lever return ERR_PTR(err); 274892476850SChuck Lever } 27492aa13531SStanislav Kinsbursky } 2750c8541ecdSChuck Lever 2751c8541ecdSChuck Lever return xprt; 2752c8541ecdSChuck Lever } 2753c8541ecdSChuck Lever 2754176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = { 2755176e21eeSChuck Lever .to_initval = 10 * HZ, 2756176e21eeSChuck Lever .to_maxval = 10 * HZ, 2757176e21eeSChuck Lever .to_retries = 2, 2758176e21eeSChuck Lever }; 2759176e21eeSChuck Lever 2760176e21eeSChuck Lever /** 2761176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket 2762176e21eeSChuck Lever * @args: rpc transport creation arguments 2763176e21eeSChuck Lever * 2764176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP 2765176e21eeSChuck Lever */ 2766176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args) 2767176e21eeSChuck Lever { 2768176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr; 2769176e21eeSChuck Lever struct sock_xprt *transport; 2770176e21eeSChuck Lever struct rpc_xprt *xprt; 2771176e21eeSChuck Lever struct rpc_xprt *ret; 2772176e21eeSChuck Lever 2773d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2774d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries); 2775176e21eeSChuck Lever if (IS_ERR(xprt)) 2776176e21eeSChuck Lever return xprt; 2777176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2778176e21eeSChuck Lever 2779176e21eeSChuck Lever xprt->prot = 0; 2780176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2781176e21eeSChuck Lever 2782176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO; 2783176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2784176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2785176e21eeSChuck Lever 2786176e21eeSChuck Lever xprt->ops = &xs_local_ops; 2787176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout; 2788176e21eeSChuck Lever 2789550aebfeSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 27904f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle); 2791550aebfeSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket); 279293dc41bdSNeilBrown 2793176e21eeSChuck Lever switch (sun->sun_family) { 2794176e21eeSChuck Lever case AF_LOCAL: 2795176e21eeSChuck Lever if (sun->sun_path[0] != '/') { 2796176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n", 2797176e21eeSChuck Lever sun->sun_path); 2798176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2799176e21eeSChuck Lever goto out_err; 2800176e21eeSChuck Lever } 2801176e21eeSChuck Lever xprt_set_bound(xprt); 2802176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL); 28037073ea87SJ. Bruce Fields ret = ERR_PTR(xs_local_setup_socket(transport)); 28047073ea87SJ. Bruce Fields if (ret) 28057073ea87SJ. Bruce Fields goto out_err; 2806176e21eeSChuck Lever break; 2807176e21eeSChuck Lever default: 2808176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT); 2809176e21eeSChuck Lever goto out_err; 2810176e21eeSChuck Lever } 2811176e21eeSChuck Lever 2812176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n", 2813176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2814176e21eeSChuck Lever 2815176e21eeSChuck Lever if (try_module_get(THIS_MODULE)) 2816176e21eeSChuck Lever return xprt; 2817176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2818176e21eeSChuck Lever out_err: 2819315f3812SKinglong Mee xs_xprt_free(xprt); 2820176e21eeSChuck Lever return ret; 2821176e21eeSChuck Lever } 2822176e21eeSChuck Lever 28232881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = { 28242881ae74STrond Myklebust .to_initval = 5 * HZ, 28252881ae74STrond Myklebust .to_maxval = 30 * HZ, 28262881ae74STrond Myklebust .to_increment = 5 * HZ, 28272881ae74STrond Myklebust .to_retries = 5, 28282881ae74STrond Myklebust }; 28292881ae74STrond Myklebust 28309903cd1cSChuck Lever /** 28319903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket 283296802a09SFrank van Maarseveen * @args: rpc transport creation arguments 28339903cd1cSChuck Lever * 28349903cd1cSChuck Lever */ 2835483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args) 2836a246b010SChuck Lever { 28378f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2838c8541ecdSChuck Lever struct rpc_xprt *xprt; 2839c8475461SChuck Lever struct sock_xprt *transport; 28400a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2841a246b010SChuck Lever 2842d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries, 2843d9ba131dSTrond Myklebust xprt_udp_slot_table_entries); 2844c8541ecdSChuck Lever if (IS_ERR(xprt)) 2845c8541ecdSChuck Lever return xprt; 2846c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2847a246b010SChuck Lever 2848ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP; 2849a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */ 2850a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3); 2851a246b010SChuck Lever 285203bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 285303bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO; 285403bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2855a246b010SChuck Lever 2856262965f5SChuck Lever xprt->ops = &xs_udp_ops; 2857a246b010SChuck Lever 2858ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout; 2859a246b010SChuck Lever 2860f9b2ee71STrond Myklebust INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn); 28614f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle); 2862edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket); 2863edc1b01cSTrond Myklebust 28648f9d5b1aSChuck Lever switch (addr->sa_family) { 28658f9d5b1aSChuck Lever case AF_INET: 28668f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 28678f9d5b1aSChuck Lever xprt_set_bound(xprt); 28688f9d5b1aSChuck Lever 28699dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP); 28708f9d5b1aSChuck Lever break; 28718f9d5b1aSChuck Lever case AF_INET6: 28728f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 28738f9d5b1aSChuck Lever xprt_set_bound(xprt); 28748f9d5b1aSChuck Lever 28759dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6); 28768f9d5b1aSChuck Lever break; 28778f9d5b1aSChuck Lever default: 28780a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 28790a68b0beSJ. Bruce Fields goto out_err; 28808f9d5b1aSChuck Lever } 28818f9d5b1aSChuck Lever 2882c740eff8SChuck Lever if (xprt_bound(xprt)) 2883c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2884c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2885c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2886c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2887c740eff8SChuck Lever else 2888c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2889c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2890c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2891edb267a6SChuck Lever 2892bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2893c8541ecdSChuck Lever return xprt; 28940a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 28950a68b0beSJ. Bruce Fields out_err: 2896315f3812SKinglong Mee xs_xprt_free(xprt); 28970a68b0beSJ. Bruce Fields return ret; 2898a246b010SChuck Lever } 2899a246b010SChuck Lever 29002881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = { 29012881ae74STrond Myklebust .to_initval = 60 * HZ, 29022881ae74STrond Myklebust .to_maxval = 60 * HZ, 29032881ae74STrond Myklebust .to_retries = 2, 29042881ae74STrond Myklebust }; 29052881ae74STrond Myklebust 29069903cd1cSChuck Lever /** 29079903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket 290896802a09SFrank van Maarseveen * @args: rpc transport creation arguments 29099903cd1cSChuck Lever * 29109903cd1cSChuck Lever */ 2911483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args) 2912a246b010SChuck Lever { 29138f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2914c8541ecdSChuck Lever struct rpc_xprt *xprt; 2915c8475461SChuck Lever struct sock_xprt *transport; 29160a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2917b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries; 2918b7993cebSTrond Myklebust 2919b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS) 2920b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT; 2921a246b010SChuck Lever 2922d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2923b7993cebSTrond Myklebust max_slot_table_size); 2924c8541ecdSChuck Lever if (IS_ERR(xprt)) 2925c8541ecdSChuck Lever return xprt; 2926c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2927a246b010SChuck Lever 2928ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP; 2929808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2930a246b010SChuck Lever 293103bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 293203bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 293303bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2934a246b010SChuck Lever 2935262965f5SChuck Lever xprt->ops = &xs_tcp_ops; 2936ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout; 2937a246b010SChuck Lever 29383851f1cdSTrond Myklebust xprt->max_reconnect_timeout = xprt->timeout->to_maxval; 29397196dbb0STrond Myklebust xprt->connect_timeout = xprt->timeout->to_initval * 29407196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 29413851f1cdSTrond Myklebust 2942c50b8ee0STrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 29434f8943f8STrond Myklebust INIT_WORK(&transport->error_worker, xs_error_handle); 2944edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket); 2945edc1b01cSTrond Myklebust 29468f9d5b1aSChuck Lever switch (addr->sa_family) { 29478f9d5b1aSChuck Lever case AF_INET: 29488f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 29498f9d5b1aSChuck Lever xprt_set_bound(xprt); 29508f9d5b1aSChuck Lever 29519dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP); 29528f9d5b1aSChuck Lever break; 29538f9d5b1aSChuck Lever case AF_INET6: 29548f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 29558f9d5b1aSChuck Lever xprt_set_bound(xprt); 29568f9d5b1aSChuck Lever 29579dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6); 29588f9d5b1aSChuck Lever break; 29598f9d5b1aSChuck Lever default: 29600a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 29610a68b0beSJ. Bruce Fields goto out_err; 29628f9d5b1aSChuck Lever } 29638f9d5b1aSChuck Lever 2964c740eff8SChuck Lever if (xprt_bound(xprt)) 2965c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2966c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2967c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2968c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2969c740eff8SChuck Lever else 2970c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2971c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2972c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2973c740eff8SChuck Lever 2974bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2975c8541ecdSChuck Lever return xprt; 29760a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29770a68b0beSJ. Bruce Fields out_err: 2978315f3812SKinglong Mee xs_xprt_free(xprt); 29790a68b0beSJ. Bruce Fields return ret; 2980a246b010SChuck Lever } 2981282b32e1SChuck Lever 2982f300babaSAlexandros Batsakis /** 2983f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket 2984f300babaSAlexandros Batsakis * @args: rpc transport creation arguments 2985f300babaSAlexandros Batsakis * 2986f300babaSAlexandros Batsakis */ 2987f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args) 2988f300babaSAlexandros Batsakis { 2989f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr; 2990f300babaSAlexandros Batsakis struct rpc_xprt *xprt; 2991f300babaSAlexandros Batsakis struct sock_xprt *transport; 2992f300babaSAlexandros Batsakis struct svc_sock *bc_sock; 29930a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2994f300babaSAlexandros Batsakis 2995d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2996d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries); 2997f300babaSAlexandros Batsakis if (IS_ERR(xprt)) 2998f300babaSAlexandros Batsakis return xprt; 2999f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt); 3000f300babaSAlexandros Batsakis 3001f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP; 3002f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 3003f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout; 3004f300babaSAlexandros Batsakis 3005f300babaSAlexandros Batsakis /* backchannel */ 3006f300babaSAlexandros Batsakis xprt_set_bound(xprt); 3007f300babaSAlexandros Batsakis xprt->bind_timeout = 0; 3008f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0; 3009f300babaSAlexandros Batsakis xprt->idle_timeout = 0; 3010f300babaSAlexandros Batsakis 3011f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops; 3012f300babaSAlexandros Batsakis 3013f300babaSAlexandros Batsakis switch (addr->sa_family) { 3014f300babaSAlexandros Batsakis case AF_INET: 3015f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3016f300babaSAlexandros Batsakis RPCBIND_NETID_TCP); 3017f300babaSAlexandros Batsakis break; 3018f300babaSAlexandros Batsakis case AF_INET6: 3019f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3020f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6); 3021f300babaSAlexandros Batsakis break; 3022f300babaSAlexandros Batsakis default: 30230a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 30240a68b0beSJ. Bruce Fields goto out_err; 3025f300babaSAlexandros Batsakis } 3026f300babaSAlexandros Batsakis 3027f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3028f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR], 3029f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT], 3030f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]); 3031f300babaSAlexandros Batsakis 3032f300babaSAlexandros Batsakis /* 303399de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection, 303428303ca3SWeng Meiling * we want to keep it around as long as the connection lasts, 303528303ca3SWeng Meiling * in case we need to start using it for a backchannel again; 303628303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed. 303799de8ea9SJ. Bruce Fields */ 303899de8ea9SJ. Bruce Fields xprt_get(xprt); 303999de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt; 304099de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt; 304199de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt); 304299de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock; 304399de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk; 304499de8ea9SJ. Bruce Fields 304599de8ea9SJ. Bruce Fields /* 3046f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set 3047f300babaSAlexandros Batsakis * the xprt status to connected 3048f300babaSAlexandros Batsakis */ 3049f300babaSAlexandros Batsakis xprt_set_connected(xprt); 3050f300babaSAlexandros Batsakis 3051f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE)) 3052f300babaSAlexandros Batsakis return xprt; 3053642aab58SKinglong Mee 3054642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL; 305539a9beabSJ. Bruce Fields args->bc_xprt->xpt_bc_xps = NULL; 305699de8ea9SJ. Bruce Fields xprt_put(xprt); 30570a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 30580a68b0beSJ. Bruce Fields out_err: 3059315f3812SKinglong Mee xs_xprt_free(xprt); 30600a68b0beSJ. Bruce Fields return ret; 3061f300babaSAlexandros Batsakis } 3062f300babaSAlexandros Batsakis 3063176e21eeSChuck Lever static struct xprt_class xs_local_transport = { 3064176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list), 3065176e21eeSChuck Lever .name = "named UNIX socket", 3066176e21eeSChuck Lever .owner = THIS_MODULE, 3067176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL, 3068176e21eeSChuck Lever .setup = xs_setup_local, 3069d5aa6b22STrond Myklebust .netid = { "" }, 3070176e21eeSChuck Lever }; 3071176e21eeSChuck Lever 3072bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = { 3073bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list), 3074bc25571eS\"Talpey, Thomas\ .name = "udp", 3075bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3076f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP, 3077bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp, 3078d5aa6b22STrond Myklebust .netid = { "udp", "udp6", "" }, 3079bc25571eS\"Talpey, Thomas\ }; 3080bc25571eS\"Talpey, Thomas\ 3081bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = { 3082bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list), 3083bc25571eS\"Talpey, Thomas\ .name = "tcp", 3084bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3085f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP, 3086bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp, 3087d5aa6b22STrond Myklebust .netid = { "tcp", "tcp6", "" }, 3088bc25571eS\"Talpey, Thomas\ }; 3089bc25571eS\"Talpey, Thomas\ 3090f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = { 3091f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list), 3092f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel", 3093f300babaSAlexandros Batsakis .owner = THIS_MODULE, 3094f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP, 3095f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp, 3096d5aa6b22STrond Myklebust .netid = { "" }, 3097f300babaSAlexandros Batsakis }; 3098f300babaSAlexandros Batsakis 3099282b32e1SChuck Lever /** 3100bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client 3101282b32e1SChuck Lever * 3102282b32e1SChuck Lever */ 3103282b32e1SChuck Lever int init_socket_xprt(void) 3104282b32e1SChuck Lever { 31052b1bec5fSEric W. Biederman if (!sunrpc_table_header) 31060b4d4147SEric W. Biederman sunrpc_table_header = register_sysctl_table(sunrpc_table); 3107fbf76683SChuck Lever 3108176e21eeSChuck Lever xprt_register_transport(&xs_local_transport); 3109bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport); 3110bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport); 3111f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport); 3112bc25571eS\"Talpey, Thomas\ 3113282b32e1SChuck Lever return 0; 3114282b32e1SChuck Lever } 3115282b32e1SChuck Lever 3116282b32e1SChuck Lever /** 3117bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister 3118282b32e1SChuck Lever * 3119282b32e1SChuck Lever */ 3120282b32e1SChuck Lever void cleanup_socket_xprt(void) 3121282b32e1SChuck Lever { 3122fbf76683SChuck Lever if (sunrpc_table_header) { 3123fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header); 3124fbf76683SChuck Lever sunrpc_table_header = NULL; 3125fbf76683SChuck Lever } 3126bc25571eS\"Talpey, Thomas\ 3127176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport); 3128bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport); 3129bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport); 3130f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport); 3131282b32e1SChuck Lever } 3132cbf11071STrond Myklebust 31339bbb9e5aSRusty Russell static int param_set_uint_minmax(const char *val, 31349bbb9e5aSRusty Russell const struct kernel_param *kp, 3135cbf11071STrond Myklebust unsigned int min, unsigned int max) 3136cbf11071STrond Myklebust { 313700cfaa94SDaniel Walter unsigned int num; 3138cbf11071STrond Myklebust int ret; 3139cbf11071STrond Myklebust 3140cbf11071STrond Myklebust if (!val) 3141cbf11071STrond Myklebust return -EINVAL; 314200cfaa94SDaniel Walter ret = kstrtouint(val, 0, &num); 31439761a246SDan Carpenter if (ret) 31449761a246SDan Carpenter return ret; 31459761a246SDan Carpenter if (num < min || num > max) 3146cbf11071STrond Myklebust return -EINVAL; 3147cbf11071STrond Myklebust *((unsigned int *)kp->arg) = num; 3148cbf11071STrond Myklebust return 0; 3149cbf11071STrond Myklebust } 3150cbf11071STrond Myklebust 31519bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp) 3152cbf11071STrond Myklebust { 3153cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3154cbf11071STrond Myklebust RPC_MIN_RESVPORT, 3155cbf11071STrond Myklebust RPC_MAX_RESVPORT); 3156cbf11071STrond Myklebust } 3157cbf11071STrond Myklebust 31589c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = { 31599bbb9e5aSRusty Russell .set = param_set_portnr, 31609bbb9e5aSRusty Russell .get = param_get_uint, 31619bbb9e5aSRusty Russell }; 31629bbb9e5aSRusty Russell 3163cbf11071STrond Myklebust #define param_check_portnr(name, p) \ 3164cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3165cbf11071STrond Myklebust 3166cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644); 3167cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644); 3168cbf11071STrond Myklebust 31699bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val, 31709bbb9e5aSRusty Russell const struct kernel_param *kp) 3171cbf11071STrond Myklebust { 3172cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3173cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE, 3174cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE); 3175cbf11071STrond Myklebust } 3176cbf11071STrond Myklebust 31779c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = { 31789bbb9e5aSRusty Russell .set = param_set_slot_table_size, 31799bbb9e5aSRusty Russell .get = param_get_uint, 31809bbb9e5aSRusty Russell }; 31819bbb9e5aSRusty Russell 3182cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \ 3183cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3184cbf11071STrond Myklebust 3185d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val, 3186d9ba131dSTrond Myklebust const struct kernel_param *kp) 3187d9ba131dSTrond Myklebust { 3188d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp, 3189d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE, 3190d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT); 3191d9ba131dSTrond Myklebust } 3192d9ba131dSTrond Myklebust 31939c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = { 3194d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size, 3195d9ba131dSTrond Myklebust .get = param_get_uint, 3196d9ba131dSTrond Myklebust }; 3197d9ba131dSTrond Myklebust 3198d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \ 3199d9ba131dSTrond Myklebust __param_check(name, p, unsigned int); 3200d9ba131dSTrond Myklebust 3201cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries, 3202cbf11071STrond Myklebust slot_table_size, 0644); 3203d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries, 3204d9ba131dSTrond Myklebust max_slot_table_size, 0644); 3205cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries, 3206cbf11071STrond Myklebust slot_table_size, 0644); 3207