1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a246b010SChuck Lever /* 3a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c 4a246b010SChuck Lever * 5a246b010SChuck Lever * Client-side transport implementation for sockets. 6a246b010SChuck Lever * 7113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat 8113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat 9a246b010SChuck Lever * TCP NFS related read + write fixes 10a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 11a246b010SChuck Lever * 12a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff. 13a246b010SChuck Lever * Fix behaviour when socket buffer is full. 14a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no> 1555aa4f58SChuck Lever * 1655aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com> 178f9d5b1aSChuck Lever * 188f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005. 198f9d5b1aSChuck Lever * <gilles.quillard@bull.net> 20a246b010SChuck Lever */ 21a246b010SChuck Lever 22a246b010SChuck Lever #include <linux/types.h> 23176e21eeSChuck Lever #include <linux/string.h> 24a246b010SChuck Lever #include <linux/slab.h> 25bc25571eS\"Talpey, Thomas\ #include <linux/module.h> 26a246b010SChuck Lever #include <linux/capability.h> 27a246b010SChuck Lever #include <linux/pagemap.h> 28a246b010SChuck Lever #include <linux/errno.h> 29a246b010SChuck Lever #include <linux/socket.h> 30a246b010SChuck Lever #include <linux/in.h> 31a246b010SChuck Lever #include <linux/net.h> 32a246b010SChuck Lever #include <linux/mm.h> 33176e21eeSChuck Lever #include <linux/un.h> 34a246b010SChuck Lever #include <linux/udp.h> 35a246b010SChuck Lever #include <linux/tcp.h> 36a246b010SChuck Lever #include <linux/sunrpc/clnt.h> 375976687aSJeff Layton #include <linux/sunrpc/addr.h> 3802107148SChuck Lever #include <linux/sunrpc/sched.h> 394cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h> 4049c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h> 41a246b010SChuck Lever #include <linux/file.h> 429e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL 4344b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h> 4444b98efdSRicardo Labiaga #endif 45a246b010SChuck Lever 46a246b010SChuck Lever #include <net/sock.h> 47a246b010SChuck Lever #include <net/checksum.h> 48a246b010SChuck Lever #include <net/udp.h> 49a246b010SChuck Lever #include <net/tcp.h> 50277e4ab7STrond Myklebust #include <linux/bvec.h> 51277e4ab7STrond Myklebust #include <linux/uio.h> 52a246b010SChuck Lever 5340b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h> 5440b5ea0cSTrond Myklebust 554cfc7e60SRahul Iyer #include "sunrpc.h" 56176e21eeSChuck Lever 57176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt); 587196dbb0STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 597196dbb0STrond Myklebust struct socket *sock); 60176e21eeSChuck Lever 619903cd1cSChuck Lever /* 62c556b754SChuck Lever * xprtsock tunables 63c556b754SChuck Lever */ 6409acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE; 6509acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE; 6609acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE; 67c556b754SChuck Lever 6809acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT; 6909acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT; 70c556b754SChuck Lever 7154c09874STrond Myklebust #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 7254c09874STrond Myklebust 737d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ) 7425fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO; 757d1e8255STrond Myklebust 76c556b754SChuck Lever /* 77fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by 78fbf76683SChuck Lever * calling register_sysctl_table() again. The files in that 79fbf76683SChuck Lever * directory become the union of all files registered there. 80fbf76683SChuck Lever * 81fbf76683SChuck Lever * We simply need to make sure that we don't collide with 82fbf76683SChuck Lever * someone else's file names! 83fbf76683SChuck Lever */ 84fbf76683SChuck Lever 85fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE; 86fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE; 87d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT; 88fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT; 89fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT; 90fbf76683SChuck Lever 91fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header; 92fbf76683SChuck Lever 93fbf76683SChuck Lever /* 94fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP 95fbf76683SChuck Lever * socket buffers for existing UDP transports 96fbf76683SChuck Lever */ 97fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = { 98fbf76683SChuck Lever { 99fbf76683SChuck Lever .procname = "udp_slot_table_entries", 100fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries, 101fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 102fbf76683SChuck Lever .mode = 0644, 1036d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 104fbf76683SChuck Lever .extra1 = &min_slot_table_size, 105fbf76683SChuck Lever .extra2 = &max_slot_table_size 106fbf76683SChuck Lever }, 107fbf76683SChuck Lever { 108fbf76683SChuck Lever .procname = "tcp_slot_table_entries", 109fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries, 110fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 111fbf76683SChuck Lever .mode = 0644, 1126d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 113fbf76683SChuck Lever .extra1 = &min_slot_table_size, 114fbf76683SChuck Lever .extra2 = &max_slot_table_size 115fbf76683SChuck Lever }, 116fbf76683SChuck Lever { 117d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries", 118d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries, 119d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int), 120d9ba131dSTrond Myklebust .mode = 0644, 121d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax, 122d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size, 123d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit 124d9ba131dSTrond Myklebust }, 125d9ba131dSTrond Myklebust { 126fbf76683SChuck Lever .procname = "min_resvport", 127fbf76683SChuck Lever .data = &xprt_min_resvport, 128fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 129fbf76683SChuck Lever .mode = 0644, 1306d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 131fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 132826799e6SJ. Bruce Fields .extra2 = &xprt_max_resvport_limit 133fbf76683SChuck Lever }, 134fbf76683SChuck Lever { 135fbf76683SChuck Lever .procname = "max_resvport", 136fbf76683SChuck Lever .data = &xprt_max_resvport, 137fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 138fbf76683SChuck Lever .mode = 0644, 1396d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 140826799e6SJ. Bruce Fields .extra1 = &xprt_min_resvport_limit, 141fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 142fbf76683SChuck Lever }, 143fbf76683SChuck Lever { 14425fe6142STrond Myklebust .procname = "tcp_fin_timeout", 14525fe6142STrond Myklebust .data = &xs_tcp_fin_timeout, 14625fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout), 14725fe6142STrond Myklebust .mode = 0644, 1486d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies, 14925fe6142STrond Myklebust }, 150f8572d8fSEric W. Biederman { }, 151fbf76683SChuck Lever }; 152fbf76683SChuck Lever 153fe2c6338SJoe Perches static struct ctl_table sunrpc_table[] = { 154fbf76683SChuck Lever { 155fbf76683SChuck Lever .procname = "sunrpc", 156fbf76683SChuck Lever .mode = 0555, 157fbf76683SChuck Lever .child = xs_tunables_table 158fbf76683SChuck Lever }, 159f8572d8fSEric W. Biederman { }, 160fbf76683SChuck Lever }; 161fbf76683SChuck Lever 162fbf76683SChuck Lever #endif 163fbf76683SChuck Lever 164fbf76683SChuck Lever /* 16503bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper. 16603bf4b70SChuck Lever */ 16703bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ) 16803bf4b70SChuck Lever 16903bf4b70SChuck Lever /* 17003bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some 17103bf4b70SChuck Lever * kind of resource problem on the local host. 17203bf4b70SChuck Lever */ 17303bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ) 17403bf4b70SChuck Lever 17503bf4b70SChuck Lever /* 17603bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting 17703bf4b70SChuck Lever * to reconnect to a server that just dropped our connection. 17803bf4b70SChuck Lever * 17903bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP 18003bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP 18103bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and 18203bf4b70SChuck Lever * increase over time if the server is down or not responding. 18303bf4b70SChuck Lever */ 18403bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ) 18503bf4b70SChuck Lever 18603bf4b70SChuck Lever /* 18703bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle 18803bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent 18903bf4b70SChuck Lever * holding port numbers when there is no RPC traffic. 19003bf4b70SChuck Lever */ 19103bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ) 19203bf4b70SChuck Lever 193f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 194a246b010SChuck Lever # undef RPC_DEBUG_DATA 1959903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS 196a246b010SChuck Lever #endif 197a246b010SChuck Lever 198a246b010SChuck Lever #ifdef RPC_DEBUG_DATA 1999903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count) 200a246b010SChuck Lever { 201a246b010SChuck Lever u8 *buf = (u8 *) packet; 202a246b010SChuck Lever int j; 203a246b010SChuck Lever 204a246b010SChuck Lever dprintk("RPC: %s\n", msg); 205a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) { 206a246b010SChuck Lever if (!(j & 31)) { 207a246b010SChuck Lever if (j) 208a246b010SChuck Lever dprintk("\n"); 209a246b010SChuck Lever dprintk("0x%04x ", j); 210a246b010SChuck Lever } 211a246b010SChuck Lever dprintk("%02x%02x%02x%02x ", 212a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]); 213a246b010SChuck Lever } 214a246b010SChuck Lever dprintk("\n"); 215a246b010SChuck Lever } 216a246b010SChuck Lever #else 2179903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) 218a246b010SChuck Lever { 219a246b010SChuck Lever /* NOP */ 220a246b010SChuck Lever } 221a246b010SChuck Lever #endif 222a246b010SChuck Lever 2232118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) 2242118071dSTrond Myklebust { 2252118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data; 2262118071dSTrond Myklebust } 2272118071dSTrond Myklebust 22895392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) 229edb267a6SChuck Lever { 23095392c59SChuck Lever return (struct sockaddr *) &xprt->addr; 23195392c59SChuck Lever } 23295392c59SChuck Lever 233176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) 234176e21eeSChuck Lever { 235176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr; 236176e21eeSChuck Lever } 237176e21eeSChuck Lever 23895392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) 23995392c59SChuck Lever { 24095392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr; 24195392c59SChuck Lever } 24295392c59SChuck Lever 24395392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) 24495392c59SChuck Lever { 24595392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr; 24695392c59SChuck Lever } 24795392c59SChuck Lever 248c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) 249c877b849SChuck Lever { 250c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2519dc3b095SChuck Lever struct sockaddr_in6 *sin6; 2529dc3b095SChuck Lever struct sockaddr_in *sin; 253176e21eeSChuck Lever struct sockaddr_un *sun; 254c877b849SChuck Lever char buf[128]; 255c877b849SChuck Lever 2569dc3b095SChuck Lever switch (sap->sa_family) { 257176e21eeSChuck Lever case AF_LOCAL: 258176e21eeSChuck Lever sun = xs_addr_un(xprt); 259176e21eeSChuck Lever strlcpy(buf, sun->sun_path, sizeof(buf)); 260176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 261176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 262176e21eeSChuck Lever break; 2639dc3b095SChuck Lever case AF_INET: 264176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 265176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 266176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2679dc3b095SChuck Lever sin = xs_addr_in(xprt); 268fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); 2699dc3b095SChuck Lever break; 2709dc3b095SChuck Lever case AF_INET6: 271176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 272176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 273176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2749dc3b095SChuck Lever sin6 = xs_addr_in6(xprt); 275fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); 2769dc3b095SChuck Lever break; 2779dc3b095SChuck Lever default: 2789dc3b095SChuck Lever BUG(); 2799dc3b095SChuck Lever } 280176e21eeSChuck Lever 2819dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); 2829dc3b095SChuck Lever } 2839dc3b095SChuck Lever 2849dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt) 2859dc3b095SChuck Lever { 2869dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2879dc3b095SChuck Lever char buf[128]; 2889dc3b095SChuck Lever 28981160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); 290c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); 291c877b849SChuck Lever 29281160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); 293c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); 294c877b849SChuck Lever } 295c877b849SChuck Lever 2969dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt, 297b454ae90SChuck Lever const char *protocol, 298b454ae90SChuck Lever const char *netid) 299edb267a6SChuck Lever { 300b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; 301b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid; 302c877b849SChuck Lever xs_format_common_peer_addresses(xprt); 3039dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 304edb267a6SChuck Lever } 305edb267a6SChuck Lever 3069dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt) 3074b6473fbSChuck Lever { 3089dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); 3099dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]); 3104b6473fbSChuck Lever 3119dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 312edb267a6SChuck Lever } 313edb267a6SChuck Lever 314edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt) 315edb267a6SChuck Lever { 31633e01dc7SChuck Lever unsigned int i; 31733e01dc7SChuck Lever 31833e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++) 31933e01dc7SChuck Lever switch (i) { 32033e01dc7SChuck Lever case RPC_DISPLAY_PROTO: 32133e01dc7SChuck Lever case RPC_DISPLAY_NETID: 32233e01dc7SChuck Lever continue; 32333e01dc7SChuck Lever default: 32433e01dc7SChuck Lever kfree(xprt->address_strings[i]); 32533e01dc7SChuck Lever } 326edb267a6SChuck Lever } 327edb267a6SChuck Lever 328277e4ab7STrond Myklebust static size_t 329277e4ab7STrond Myklebust xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) 330277e4ab7STrond Myklebust { 331277e4ab7STrond Myklebust size_t i,n; 332277e4ab7STrond Myklebust 33316e5e90fSTrond Myklebust if (!want || !(buf->flags & XDRBUF_SPARSE_PAGES)) 334277e4ab7STrond Myklebust return want; 335277e4ab7STrond Myklebust n = (buf->page_base + want + PAGE_SIZE - 1) >> PAGE_SHIFT; 336277e4ab7STrond Myklebust for (i = 0; i < n; i++) { 337277e4ab7STrond Myklebust if (buf->pages[i]) 338277e4ab7STrond Myklebust continue; 339277e4ab7STrond Myklebust buf->bvec[i].bv_page = buf->pages[i] = alloc_page(gfp); 340277e4ab7STrond Myklebust if (!buf->pages[i]) { 34116e5e90fSTrond Myklebust i *= PAGE_SIZE; 34216e5e90fSTrond Myklebust return i > buf->page_base ? i - buf->page_base : 0; 343277e4ab7STrond Myklebust } 344277e4ab7STrond Myklebust } 345277e4ab7STrond Myklebust return want; 346277e4ab7STrond Myklebust } 347277e4ab7STrond Myklebust 348277e4ab7STrond Myklebust static ssize_t 349277e4ab7STrond Myklebust xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) 350277e4ab7STrond Myklebust { 351277e4ab7STrond Myklebust ssize_t ret; 352277e4ab7STrond Myklebust if (seek != 0) 353277e4ab7STrond Myklebust iov_iter_advance(&msg->msg_iter, seek); 354277e4ab7STrond Myklebust ret = sock_recvmsg(sock, msg, flags); 355277e4ab7STrond Myklebust return ret > 0 ? ret + seek : ret; 356277e4ab7STrond Myklebust } 357277e4ab7STrond Myklebust 358277e4ab7STrond Myklebust static ssize_t 359277e4ab7STrond Myklebust xs_read_kvec(struct socket *sock, struct msghdr *msg, int flags, 360277e4ab7STrond Myklebust struct kvec *kvec, size_t count, size_t seek) 361277e4ab7STrond Myklebust { 3620e9b4a82SAl Viro iov_iter_kvec(&msg->msg_iter, READ, kvec, 1, count); 363277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 364277e4ab7STrond Myklebust } 365277e4ab7STrond Myklebust 366277e4ab7STrond Myklebust static ssize_t 367277e4ab7STrond Myklebust xs_read_bvec(struct socket *sock, struct msghdr *msg, int flags, 368277e4ab7STrond Myklebust struct bio_vec *bvec, unsigned long nr, size_t count, 369277e4ab7STrond Myklebust size_t seek) 370277e4ab7STrond Myklebust { 3710e9b4a82SAl Viro iov_iter_bvec(&msg->msg_iter, READ, bvec, nr, count); 372277e4ab7STrond Myklebust return xs_sock_recvmsg(sock, msg, flags, seek); 373277e4ab7STrond Myklebust } 374277e4ab7STrond Myklebust 375277e4ab7STrond Myklebust static ssize_t 376277e4ab7STrond Myklebust xs_read_discard(struct socket *sock, struct msghdr *msg, int flags, 377277e4ab7STrond Myklebust size_t count) 378277e4ab7STrond Myklebust { 379b76a5afdSTrond Myklebust iov_iter_discard(&msg->msg_iter, READ, count); 380b76a5afdSTrond Myklebust return sock_recvmsg(sock, msg, flags); 381277e4ab7STrond Myklebust } 382277e4ab7STrond Myklebust 383277e4ab7STrond Myklebust static ssize_t 384277e4ab7STrond Myklebust xs_read_xdr_buf(struct socket *sock, struct msghdr *msg, int flags, 385277e4ab7STrond Myklebust struct xdr_buf *buf, size_t count, size_t seek, size_t *read) 386277e4ab7STrond Myklebust { 387277e4ab7STrond Myklebust size_t want, seek_init = seek, offset = 0; 388277e4ab7STrond Myklebust ssize_t ret; 389277e4ab7STrond Myklebust 390277e4ab7STrond Myklebust if (seek < buf->head[0].iov_len) { 391277e4ab7STrond Myklebust want = min_t(size_t, count, buf->head[0].iov_len); 392277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->head[0], want, seek); 393277e4ab7STrond Myklebust if (ret <= 0) 394277e4ab7STrond Myklebust goto sock_err; 395277e4ab7STrond Myklebust offset += ret; 396277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 397277e4ab7STrond Myklebust goto out; 398277e4ab7STrond Myklebust if (ret != want) 399c4433055STrond Myklebust goto out; 400277e4ab7STrond Myklebust seek = 0; 401277e4ab7STrond Myklebust } else { 402277e4ab7STrond Myklebust seek -= buf->head[0].iov_len; 403277e4ab7STrond Myklebust offset += buf->head[0].iov_len; 404277e4ab7STrond Myklebust } 40516e5e90fSTrond Myklebust 406277e4ab7STrond Myklebust want = xs_alloc_sparse_pages(buf, 407277e4ab7STrond Myklebust min_t(size_t, count - offset, buf->page_len), 408277e4ab7STrond Myklebust GFP_NOWAIT); 40916e5e90fSTrond Myklebust if (seek < want) { 410277e4ab7STrond Myklebust ret = xs_read_bvec(sock, msg, flags, buf->bvec, 411277e4ab7STrond Myklebust xdr_buf_pagecount(buf), 412277e4ab7STrond Myklebust want + buf->page_base, 413277e4ab7STrond Myklebust seek + buf->page_base); 414277e4ab7STrond Myklebust if (ret <= 0) 415277e4ab7STrond Myklebust goto sock_err; 416277e4ab7STrond Myklebust offset += ret - buf->page_base; 417277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 418277e4ab7STrond Myklebust goto out; 419277e4ab7STrond Myklebust if (ret != want) 420c4433055STrond Myklebust goto out; 421277e4ab7STrond Myklebust seek = 0; 422277e4ab7STrond Myklebust } else { 42316e5e90fSTrond Myklebust seek -= want; 42416e5e90fSTrond Myklebust offset += want; 425277e4ab7STrond Myklebust } 42616e5e90fSTrond Myklebust 427277e4ab7STrond Myklebust if (seek < buf->tail[0].iov_len) { 428277e4ab7STrond Myklebust want = min_t(size_t, count - offset, buf->tail[0].iov_len); 429277e4ab7STrond Myklebust ret = xs_read_kvec(sock, msg, flags, &buf->tail[0], want, seek); 430277e4ab7STrond Myklebust if (ret <= 0) 431277e4ab7STrond Myklebust goto sock_err; 432277e4ab7STrond Myklebust offset += ret; 433277e4ab7STrond Myklebust if (offset == count || msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 434277e4ab7STrond Myklebust goto out; 435277e4ab7STrond Myklebust if (ret != want) 436c4433055STrond Myklebust goto out; 437277e4ab7STrond Myklebust } else 438277e4ab7STrond Myklebust offset += buf->tail[0].iov_len; 439277e4ab7STrond Myklebust ret = -EMSGSIZE; 440277e4ab7STrond Myklebust out: 441277e4ab7STrond Myklebust *read = offset - seek_init; 442277e4ab7STrond Myklebust return ret; 443277e4ab7STrond Myklebust sock_err: 444277e4ab7STrond Myklebust offset += seek; 445277e4ab7STrond Myklebust goto out; 446277e4ab7STrond Myklebust } 447277e4ab7STrond Myklebust 448277e4ab7STrond Myklebust static void 449277e4ab7STrond Myklebust xs_read_header(struct sock_xprt *transport, struct xdr_buf *buf) 450277e4ab7STrond Myklebust { 451277e4ab7STrond Myklebust if (!transport->recv.copied) { 452277e4ab7STrond Myklebust if (buf->head[0].iov_len >= transport->recv.offset) 453277e4ab7STrond Myklebust memcpy(buf->head[0].iov_base, 454277e4ab7STrond Myklebust &transport->recv.xid, 455277e4ab7STrond Myklebust transport->recv.offset); 456277e4ab7STrond Myklebust transport->recv.copied = transport->recv.offset; 457277e4ab7STrond Myklebust } 458277e4ab7STrond Myklebust } 459277e4ab7STrond Myklebust 460277e4ab7STrond Myklebust static bool 461277e4ab7STrond Myklebust xs_read_stream_request_done(struct sock_xprt *transport) 462277e4ab7STrond Myklebust { 463277e4ab7STrond Myklebust return transport->recv.fraghdr & cpu_to_be32(RPC_LAST_STREAM_FRAGMENT); 464277e4ab7STrond Myklebust } 465277e4ab7STrond Myklebust 466277e4ab7STrond Myklebust static ssize_t 467277e4ab7STrond Myklebust xs_read_stream_request(struct sock_xprt *transport, struct msghdr *msg, 468277e4ab7STrond Myklebust int flags, struct rpc_rqst *req) 469277e4ab7STrond Myklebust { 470277e4ab7STrond Myklebust struct xdr_buf *buf = &req->rq_private_buf; 471277e4ab7STrond Myklebust size_t want, read; 472277e4ab7STrond Myklebust ssize_t ret; 473277e4ab7STrond Myklebust 474277e4ab7STrond Myklebust xs_read_header(transport, buf); 475277e4ab7STrond Myklebust 476277e4ab7STrond Myklebust want = transport->recv.len - transport->recv.offset; 477277e4ab7STrond Myklebust ret = xs_read_xdr_buf(transport->sock, msg, flags, buf, 478277e4ab7STrond Myklebust transport->recv.copied + want, transport->recv.copied, 479277e4ab7STrond Myklebust &read); 480277e4ab7STrond Myklebust transport->recv.offset += read; 481277e4ab7STrond Myklebust transport->recv.copied += read; 482277e4ab7STrond Myklebust if (transport->recv.offset == transport->recv.len) { 483277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) 484277e4ab7STrond Myklebust msg->msg_flags |= MSG_EOR; 485c4433055STrond Myklebust return read; 486277e4ab7STrond Myklebust } 487277e4ab7STrond Myklebust 488277e4ab7STrond Myklebust switch (ret) { 489c4433055STrond Myklebust default: 490c4433055STrond Myklebust break; 49126781eabSTrond Myklebust case -EFAULT: 492277e4ab7STrond Myklebust case -EMSGSIZE: 49326781eabSTrond Myklebust msg->msg_flags |= MSG_TRUNC; 494c4433055STrond Myklebust return read; 495277e4ab7STrond Myklebust case 0: 496277e4ab7STrond Myklebust return -ESHUTDOWN; 497277e4ab7STrond Myklebust } 498c4433055STrond Myklebust return ret < 0 ? ret : read; 499277e4ab7STrond Myklebust } 500277e4ab7STrond Myklebust 501277e4ab7STrond Myklebust static size_t 502277e4ab7STrond Myklebust xs_read_stream_headersize(bool isfrag) 503277e4ab7STrond Myklebust { 504277e4ab7STrond Myklebust if (isfrag) 505277e4ab7STrond Myklebust return sizeof(__be32); 506277e4ab7STrond Myklebust return 3 * sizeof(__be32); 507277e4ab7STrond Myklebust } 508277e4ab7STrond Myklebust 509277e4ab7STrond Myklebust static ssize_t 510277e4ab7STrond Myklebust xs_read_stream_header(struct sock_xprt *transport, struct msghdr *msg, 511277e4ab7STrond Myklebust int flags, size_t want, size_t seek) 512277e4ab7STrond Myklebust { 513277e4ab7STrond Myklebust struct kvec kvec = { 514277e4ab7STrond Myklebust .iov_base = &transport->recv.fraghdr, 515277e4ab7STrond Myklebust .iov_len = want, 516277e4ab7STrond Myklebust }; 517277e4ab7STrond Myklebust return xs_read_kvec(transport->sock, msg, flags, &kvec, want, seek); 518277e4ab7STrond Myklebust } 519277e4ab7STrond Myklebust 520277e4ab7STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 521277e4ab7STrond Myklebust static ssize_t 522277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 523277e4ab7STrond Myklebust { 524277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 525277e4ab7STrond Myklebust struct rpc_rqst *req; 526277e4ab7STrond Myklebust ssize_t ret; 527277e4ab7STrond Myklebust 528277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 529277e4ab7STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->recv.xid); 530277e4ab7STrond Myklebust if (!req) { 531277e4ab7STrond Myklebust printk(KERN_WARNING "Callback slot table overflowed\n"); 532277e4ab7STrond Myklebust return -ESHUTDOWN; 533277e4ab7STrond Myklebust } 534277e4ab7STrond Myklebust 535277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 536277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 537c4433055STrond Myklebust xprt_complete_bc_request(req, transport->recv.copied); 538277e4ab7STrond Myklebust 539277e4ab7STrond Myklebust return ret; 540277e4ab7STrond Myklebust } 541277e4ab7STrond Myklebust #else /* CONFIG_SUNRPC_BACKCHANNEL */ 542277e4ab7STrond Myklebust static ssize_t 543277e4ab7STrond Myklebust xs_read_stream_call(struct sock_xprt *transport, struct msghdr *msg, int flags) 544277e4ab7STrond Myklebust { 545277e4ab7STrond Myklebust return -ESHUTDOWN; 546277e4ab7STrond Myklebust } 547277e4ab7STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 548277e4ab7STrond Myklebust 549277e4ab7STrond Myklebust static ssize_t 550277e4ab7STrond Myklebust xs_read_stream_reply(struct sock_xprt *transport, struct msghdr *msg, int flags) 551277e4ab7STrond Myklebust { 552277e4ab7STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 553277e4ab7STrond Myklebust struct rpc_rqst *req; 554277e4ab7STrond Myklebust ssize_t ret = 0; 555277e4ab7STrond Myklebust 556277e4ab7STrond Myklebust /* Look up and lock the request corresponding to the given XID */ 557277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 558277e4ab7STrond Myklebust req = xprt_lookup_rqst(xprt, transport->recv.xid); 559277e4ab7STrond Myklebust if (!req) { 560277e4ab7STrond Myklebust msg->msg_flags |= MSG_TRUNC; 561277e4ab7STrond Myklebust goto out; 562277e4ab7STrond Myklebust } 563277e4ab7STrond Myklebust xprt_pin_rqst(req); 564277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 565277e4ab7STrond Myklebust 566277e4ab7STrond Myklebust ret = xs_read_stream_request(transport, msg, flags, req); 567277e4ab7STrond Myklebust 568277e4ab7STrond Myklebust spin_lock(&xprt->queue_lock); 569277e4ab7STrond Myklebust if (msg->msg_flags & (MSG_EOR|MSG_TRUNC)) 570c4433055STrond Myklebust xprt_complete_rqst(req->rq_task, transport->recv.copied); 571277e4ab7STrond Myklebust xprt_unpin_rqst(req); 572277e4ab7STrond Myklebust out: 573277e4ab7STrond Myklebust spin_unlock(&xprt->queue_lock); 574277e4ab7STrond Myklebust return ret; 575277e4ab7STrond Myklebust } 576277e4ab7STrond Myklebust 577277e4ab7STrond Myklebust static ssize_t 578277e4ab7STrond Myklebust xs_read_stream(struct sock_xprt *transport, int flags) 579277e4ab7STrond Myklebust { 580277e4ab7STrond Myklebust struct msghdr msg = { 0 }; 581277e4ab7STrond Myklebust size_t want, read = 0; 582277e4ab7STrond Myklebust ssize_t ret = 0; 583277e4ab7STrond Myklebust 584277e4ab7STrond Myklebust if (transport->recv.len == 0) { 585277e4ab7STrond Myklebust want = xs_read_stream_headersize(transport->recv.copied != 0); 586277e4ab7STrond Myklebust ret = xs_read_stream_header(transport, &msg, flags, want, 587277e4ab7STrond Myklebust transport->recv.offset); 588277e4ab7STrond Myklebust if (ret <= 0) 589277e4ab7STrond Myklebust goto out_err; 590277e4ab7STrond Myklebust transport->recv.offset = ret; 591c4433055STrond Myklebust if (transport->recv.offset != want) 592c4433055STrond Myklebust return transport->recv.offset; 593277e4ab7STrond Myklebust transport->recv.len = be32_to_cpu(transport->recv.fraghdr) & 594277e4ab7STrond Myklebust RPC_FRAGMENT_SIZE_MASK; 595277e4ab7STrond Myklebust transport->recv.offset -= sizeof(transport->recv.fraghdr); 596277e4ab7STrond Myklebust read = ret; 597277e4ab7STrond Myklebust } 598277e4ab7STrond Myklebust 599277e4ab7STrond Myklebust switch (be32_to_cpu(transport->recv.calldir)) { 600c4433055STrond Myklebust default: 601c4433055STrond Myklebust msg.msg_flags |= MSG_TRUNC; 602c4433055STrond Myklebust break; 603277e4ab7STrond Myklebust case RPC_CALL: 604277e4ab7STrond Myklebust ret = xs_read_stream_call(transport, &msg, flags); 605277e4ab7STrond Myklebust break; 606277e4ab7STrond Myklebust case RPC_REPLY: 607277e4ab7STrond Myklebust ret = xs_read_stream_reply(transport, &msg, flags); 608277e4ab7STrond Myklebust } 609277e4ab7STrond Myklebust if (msg.msg_flags & MSG_TRUNC) { 610277e4ab7STrond Myklebust transport->recv.calldir = cpu_to_be32(-1); 611277e4ab7STrond Myklebust transport->recv.copied = -1; 612277e4ab7STrond Myklebust } 613277e4ab7STrond Myklebust if (ret < 0) 614277e4ab7STrond Myklebust goto out_err; 615277e4ab7STrond Myklebust read += ret; 616277e4ab7STrond Myklebust if (transport->recv.offset < transport->recv.len) { 617c4433055STrond Myklebust if (!(msg.msg_flags & MSG_TRUNC)) 618c4433055STrond Myklebust return read; 619b76a5afdSTrond Myklebust msg.msg_flags = 0; 620277e4ab7STrond Myklebust ret = xs_read_discard(transport->sock, &msg, flags, 621277e4ab7STrond Myklebust transport->recv.len - transport->recv.offset); 622277e4ab7STrond Myklebust if (ret <= 0) 623277e4ab7STrond Myklebust goto out_err; 624277e4ab7STrond Myklebust transport->recv.offset += ret; 625277e4ab7STrond Myklebust read += ret; 626277e4ab7STrond Myklebust if (transport->recv.offset != transport->recv.len) 627c4433055STrond Myklebust return read; 628277e4ab7STrond Myklebust } 629277e4ab7STrond Myklebust if (xs_read_stream_request_done(transport)) { 630c50b8ee0STrond Myklebust trace_xs_stream_read_request(transport); 631277e4ab7STrond Myklebust transport->recv.copied = 0; 632277e4ab7STrond Myklebust } 633277e4ab7STrond Myklebust transport->recv.offset = 0; 634277e4ab7STrond Myklebust transport->recv.len = 0; 635277e4ab7STrond Myklebust return read; 636277e4ab7STrond Myklebust out_err: 637277e4ab7STrond Myklebust switch (ret) { 638277e4ab7STrond Myklebust case 0: 639277e4ab7STrond Myklebust case -ESHUTDOWN: 640277e4ab7STrond Myklebust xprt_force_disconnect(&transport->xprt); 641277e4ab7STrond Myklebust return -ESHUTDOWN; 642277e4ab7STrond Myklebust } 643277e4ab7STrond Myklebust return ret; 644277e4ab7STrond Myklebust } 645277e4ab7STrond Myklebust 646c50b8ee0STrond Myklebust static void xs_stream_data_receive(struct sock_xprt *transport) 647c50b8ee0STrond Myklebust { 648c50b8ee0STrond Myklebust size_t read = 0; 649c50b8ee0STrond Myklebust ssize_t ret = 0; 650c50b8ee0STrond Myklebust 651c50b8ee0STrond Myklebust mutex_lock(&transport->recv_mutex); 652c50b8ee0STrond Myklebust if (transport->sock == NULL) 653c50b8ee0STrond Myklebust goto out; 654c50b8ee0STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 655c50b8ee0STrond Myklebust for (;;) { 656c50b8ee0STrond Myklebust ret = xs_read_stream(transport, MSG_DONTWAIT); 657c4433055STrond Myklebust if (ret < 0) 658c50b8ee0STrond Myklebust break; 659c50b8ee0STrond Myklebust read += ret; 660c50b8ee0STrond Myklebust cond_resched(); 661c50b8ee0STrond Myklebust } 662c50b8ee0STrond Myklebust out: 663c50b8ee0STrond Myklebust mutex_unlock(&transport->recv_mutex); 664c50b8ee0STrond Myklebust trace_xs_stream_read_data(&transport->xprt, ret, read); 665c50b8ee0STrond Myklebust } 666c50b8ee0STrond Myklebust 667c50b8ee0STrond Myklebust static void xs_stream_data_receive_workfn(struct work_struct *work) 668c50b8ee0STrond Myklebust { 669c50b8ee0STrond Myklebust struct sock_xprt *transport = 670c50b8ee0STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 671c50b8ee0STrond Myklebust xs_stream_data_receive(transport); 672c50b8ee0STrond Myklebust } 673c50b8ee0STrond Myklebust 674550aebfeSTrond Myklebust static void 675550aebfeSTrond Myklebust xs_stream_reset_connect(struct sock_xprt *transport) 676550aebfeSTrond Myklebust { 677550aebfeSTrond Myklebust transport->recv.offset = 0; 678550aebfeSTrond Myklebust transport->recv.len = 0; 679550aebfeSTrond Myklebust transport->recv.copied = 0; 680550aebfeSTrond Myklebust transport->xmit.offset = 0; 681550aebfeSTrond Myklebust transport->xprt.stat.connect_count++; 682550aebfeSTrond Myklebust transport->xprt.stat.connect_start = jiffies; 683550aebfeSTrond Myklebust } 684550aebfeSTrond Myklebust 685b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 686b4b5cc85SChuck Lever 68724c5684bSTrond Myklebust static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more) 688b4b5cc85SChuck Lever { 689b4b5cc85SChuck Lever struct msghdr msg = { 690b4b5cc85SChuck Lever .msg_name = addr, 691b4b5cc85SChuck Lever .msg_namelen = addrlen, 69224c5684bSTrond Myklebust .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0), 69324c5684bSTrond Myklebust }; 69424c5684bSTrond Myklebust struct kvec iov = { 69524c5684bSTrond Myklebust .iov_base = vec->iov_base + base, 69624c5684bSTrond Myklebust .iov_len = vec->iov_len - base, 697b4b5cc85SChuck Lever }; 698b4b5cc85SChuck Lever 69924c5684bSTrond Myklebust if (iov.iov_len != 0) 700b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); 701b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, NULL, 0, 0); 702b4b5cc85SChuck Lever } 703b4b5cc85SChuck Lever 704f279cd00SJason Baron static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p) 705b4b5cc85SChuck Lever { 706a6b31d18STrond Myklebust ssize_t (*do_sendpage)(struct socket *sock, struct page *page, 707a6b31d18STrond Myklebust int offset, size_t size, int flags); 70824c5684bSTrond Myklebust struct page **ppage; 70924c5684bSTrond Myklebust unsigned int remainder; 710f279cd00SJason Baron int err; 711b4b5cc85SChuck Lever 71224c5684bSTrond Myklebust remainder = xdr->page_len - base; 71324c5684bSTrond Myklebust base += xdr->page_base; 71424c5684bSTrond Myklebust ppage = xdr->pages + (base >> PAGE_SHIFT); 71524c5684bSTrond Myklebust base &= ~PAGE_MASK; 716a6b31d18STrond Myklebust do_sendpage = sock->ops->sendpage; 717a6b31d18STrond Myklebust if (!zerocopy) 718a6b31d18STrond Myklebust do_sendpage = sock_no_sendpage; 71924c5684bSTrond Myklebust for(;;) { 72024c5684bSTrond Myklebust unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder); 72124c5684bSTrond Myklebust int flags = XS_SENDMSG_FLAGS; 72224c5684bSTrond Myklebust 72324c5684bSTrond Myklebust remainder -= len; 72431303d6cSTrond Myklebust if (more) 72524c5684bSTrond Myklebust flags |= MSG_MORE; 72631303d6cSTrond Myklebust if (remainder != 0) 72731303d6cSTrond Myklebust flags |= MSG_SENDPAGE_NOTLAST | MSG_MORE; 728a6b31d18STrond Myklebust err = do_sendpage(sock, *ppage, base, len, flags); 72924c5684bSTrond Myklebust if (remainder == 0 || err != len) 73024c5684bSTrond Myklebust break; 731f279cd00SJason Baron *sent_p += err; 73224c5684bSTrond Myklebust ppage++; 73324c5684bSTrond Myklebust base = 0; 73424c5684bSTrond Myklebust } 735f279cd00SJason Baron if (err > 0) { 736f279cd00SJason Baron *sent_p += err; 737f279cd00SJason Baron err = 0; 738f279cd00SJason Baron } 73924c5684bSTrond Myklebust return err; 740b4b5cc85SChuck Lever } 741b4b5cc85SChuck Lever 7429903cd1cSChuck Lever /** 7439903cd1cSChuck Lever * xs_sendpages - write pages directly to a socket 7449903cd1cSChuck Lever * @sock: socket to send on 7459903cd1cSChuck Lever * @addr: UDP only -- address of destination 7469903cd1cSChuck Lever * @addrlen: UDP only -- length of destination address 7479903cd1cSChuck Lever * @xdr: buffer containing this request 7489903cd1cSChuck Lever * @base: starting position in the buffer 749a6b31d18STrond Myklebust * @zerocopy: true if it is safe to use sendpage() 750f279cd00SJason Baron * @sent_p: return the total number of bytes successfully queued for sending 7519903cd1cSChuck Lever * 752a246b010SChuck Lever */ 753f279cd00SJason Baron static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p) 754a246b010SChuck Lever { 75524c5684bSTrond Myklebust unsigned int remainder = xdr->len - base; 756f279cd00SJason Baron int err = 0; 757f279cd00SJason Baron int sent = 0; 758a246b010SChuck Lever 759262965f5SChuck Lever if (unlikely(!sock)) 760fba91afbSTrond Myklebust return -ENOTSOCK; 761262965f5SChuck Lever 76224c5684bSTrond Myklebust if (base != 0) { 76324c5684bSTrond Myklebust addr = NULL; 76424c5684bSTrond Myklebust addrlen = 0; 76524c5684bSTrond Myklebust } 766262965f5SChuck Lever 76724c5684bSTrond Myklebust if (base < xdr->head[0].iov_len || addr != NULL) { 76824c5684bSTrond Myklebust unsigned int len = xdr->head[0].iov_len - base; 76924c5684bSTrond Myklebust remainder -= len; 77024c5684bSTrond Myklebust err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0); 77124c5684bSTrond Myklebust if (remainder == 0 || err != len) 772a246b010SChuck Lever goto out; 773f279cd00SJason Baron *sent_p += err; 774a246b010SChuck Lever base = 0; 775a246b010SChuck Lever } else 77624c5684bSTrond Myklebust base -= xdr->head[0].iov_len; 777a246b010SChuck Lever 77824c5684bSTrond Myklebust if (base < xdr->page_len) { 77924c5684bSTrond Myklebust unsigned int len = xdr->page_len - base; 78024c5684bSTrond Myklebust remainder -= len; 781f279cd00SJason Baron err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent); 782f279cd00SJason Baron *sent_p += sent; 783f279cd00SJason Baron if (remainder == 0 || sent != len) 784a246b010SChuck Lever goto out; 785a246b010SChuck Lever base = 0; 78624c5684bSTrond Myklebust } else 78724c5684bSTrond Myklebust base -= xdr->page_len; 78824c5684bSTrond Myklebust 78924c5684bSTrond Myklebust if (base >= xdr->tail[0].iov_len) 790f279cd00SJason Baron return 0; 79124c5684bSTrond Myklebust err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0); 792a246b010SChuck Lever out: 793f279cd00SJason Baron if (err > 0) { 794f279cd00SJason Baron *sent_p += err; 795f279cd00SJason Baron err = 0; 796f279cd00SJason Baron } 79724c5684bSTrond Myklebust return err; 798a246b010SChuck Lever } 799a246b010SChuck Lever 8009903cd1cSChuck Lever /** 801c544577dSTrond Myklebust * xs_nospace - handle transmit was incomplete 80250f484e2STrond Myklebust * @req: pointer to RPC request 8039903cd1cSChuck Lever * 804a246b010SChuck Lever */ 805c544577dSTrond Myklebust static int xs_nospace(struct rpc_rqst *req) 806a246b010SChuck Lever { 807262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 808ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 80906ea0bfeSTrond Myklebust struct sock *sk = transport->inet; 81024ca9a84STrond Myklebust int ret = -EAGAIN; 811a246b010SChuck Lever 81246121cf7SChuck Lever dprintk("RPC: %5u xmit incomplete (%u left of %u)\n", 813c544577dSTrond Myklebust req->rq_task->tk_pid, 814c544577dSTrond Myklebust req->rq_slen - transport->xmit.offset, 815262965f5SChuck Lever req->rq_slen); 816a246b010SChuck Lever 817262965f5SChuck Lever /* Protect against races with write_space */ 818262965f5SChuck Lever spin_lock_bh(&xprt->transport_lock); 819a246b010SChuck Lever 820262965f5SChuck Lever /* Don't race with disconnect */ 821b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) { 82213331a55STrond Myklebust /* wait for more buffer space */ 82306ea0bfeSTrond Myklebust sk->sk_write_pending++; 824c544577dSTrond Myklebust xprt_wait_for_buffer_space(xprt); 82513331a55STrond Myklebust } else 8265e3771ceSTrond Myklebust ret = -ENOTCONN; 827a246b010SChuck Lever 828262965f5SChuck Lever spin_unlock_bh(&xprt->transport_lock); 82906ea0bfeSTrond Myklebust 83006ea0bfeSTrond Myklebust /* Race breaker in case memory is freed before above code is called */ 831d48f9ce7SDavid Vrabel if (ret == -EAGAIN) { 832d48f9ce7SDavid Vrabel struct socket_wq *wq; 833d48f9ce7SDavid Vrabel 834d48f9ce7SDavid Vrabel rcu_read_lock(); 835d48f9ce7SDavid Vrabel wq = rcu_dereference(sk->sk_wq); 836d48f9ce7SDavid Vrabel set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags); 837d48f9ce7SDavid Vrabel rcu_read_unlock(); 838d48f9ce7SDavid Vrabel 83906ea0bfeSTrond Myklebust sk->sk_write_space(sk); 840d48f9ce7SDavid Vrabel } 8415e3771ceSTrond Myklebust return ret; 842a246b010SChuck Lever } 843a246b010SChuck Lever 844277e4ab7STrond Myklebust static void 845277e4ab7STrond Myklebust xs_stream_prepare_request(struct rpc_rqst *req) 846277e4ab7STrond Myklebust { 847277e4ab7STrond Myklebust req->rq_task->tk_status = xdr_alloc_bvec(&req->rq_rcv_buf, GFP_NOIO); 848277e4ab7STrond Myklebust } 849277e4ab7STrond Myklebust 85061677eeeSChuck Lever /* 8514cd34e7cSTrond Myklebust * Determine if the previous message in the stream was aborted before it 8524cd34e7cSTrond Myklebust * could complete transmission. 8534cd34e7cSTrond Myklebust */ 8544cd34e7cSTrond Myklebust static bool 8554cd34e7cSTrond Myklebust xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req) 8564cd34e7cSTrond Myklebust { 8574cd34e7cSTrond Myklebust return transport->xmit.offset != 0 && req->rq_bytes_sent == 0; 8584cd34e7cSTrond Myklebust } 8594cd34e7cSTrond Myklebust 8604cd34e7cSTrond Myklebust /* 86161677eeeSChuck Lever * Construct a stream transport record marker in @buf. 86261677eeeSChuck Lever */ 86361677eeeSChuck Lever static inline void xs_encode_stream_record_marker(struct xdr_buf *buf) 86461677eeeSChuck Lever { 86561677eeeSChuck Lever u32 reclen = buf->len - sizeof(rpc_fraghdr); 86661677eeeSChuck Lever rpc_fraghdr *base = buf->head[0].iov_base; 86761677eeeSChuck Lever *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen); 86861677eeeSChuck Lever } 86961677eeeSChuck Lever 8709903cd1cSChuck Lever /** 871176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket 87250f484e2STrond Myklebust * @req: pointer to RPC request 873176e21eeSChuck Lever * 874176e21eeSChuck Lever * Return values: 875176e21eeSChuck Lever * 0: The request has been sent 876176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to 877176e21eeSChuck Lever * complete the request 878176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 879176e21eeSChuck Lever * other: Some other error occured, the request was not sent 880176e21eeSChuck Lever */ 881adfa7144STrond Myklebust static int xs_local_send_request(struct rpc_rqst *req) 882176e21eeSChuck Lever { 883176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 884176e21eeSChuck Lever struct sock_xprt *transport = 885176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt); 886176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 887176e21eeSChuck Lever int status; 888f279cd00SJason Baron int sent = 0; 889176e21eeSChuck Lever 8904cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 8914cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 8924cd34e7cSTrond Myklebust xs_close(xprt); 8934cd34e7cSTrond Myklebust return -ENOTCONN; 8944cd34e7cSTrond Myklebust } 8954cd34e7cSTrond Myklebust 896176e21eeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 897176e21eeSChuck Lever 898176e21eeSChuck Lever xs_pktdump("packet data:", 899176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len); 900176e21eeSChuck Lever 90178215759SChuck Lever req->rq_xtime = ktime_get(); 9026c7a64e5STrond Myklebust status = xs_sendpages(transport->sock, NULL, 0, xdr, 9036c7a64e5STrond Myklebust transport->xmit.offset, 904f279cd00SJason Baron true, &sent); 905176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n", 9066c7a64e5STrond Myklebust __func__, xdr->len - transport->xmit.offset, status); 907743c69e7SNeilBrown 908743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 909743c69e7SNeilBrown status = -ENOBUFS; 910743c69e7SNeilBrown 911f279cd00SJason Baron if (likely(sent > 0) || status == 0) { 9126c7a64e5STrond Myklebust transport->xmit.offset += sent; 9136c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 914176e21eeSChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 9156c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 916176e21eeSChuck Lever req->rq_bytes_sent = 0; 9176c7a64e5STrond Myklebust transport->xmit.offset = 0; 918176e21eeSChuck Lever return 0; 919176e21eeSChuck Lever } 920176e21eeSChuck Lever status = -EAGAIN; 921176e21eeSChuck Lever } 922176e21eeSChuck Lever 923176e21eeSChuck Lever switch (status) { 9243601c4a9STrond Myklebust case -ENOBUFS: 925b5872f0cSTrond Myklebust break; 926176e21eeSChuck Lever case -EAGAIN: 927c544577dSTrond Myklebust status = xs_nospace(req); 928176e21eeSChuck Lever break; 929176e21eeSChuck Lever default: 930176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 931176e21eeSChuck Lever -status); 932e9d47639SGustavo A. R. Silva /* fall through */ 933176e21eeSChuck Lever case -EPIPE: 934176e21eeSChuck Lever xs_close(xprt); 935176e21eeSChuck Lever status = -ENOTCONN; 936176e21eeSChuck Lever } 937176e21eeSChuck Lever 938176e21eeSChuck Lever return status; 939176e21eeSChuck Lever } 940176e21eeSChuck Lever 941176e21eeSChuck Lever /** 942262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket 94350f484e2STrond Myklebust * @req: pointer to RPC request 9449903cd1cSChuck Lever * 9459903cd1cSChuck Lever * Return values: 9469903cd1cSChuck Lever * 0: The request has been sent 9479903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to 9489903cd1cSChuck Lever * complete the request 949262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 95025985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 9519903cd1cSChuck Lever */ 952adfa7144STrond Myklebust static int xs_udp_send_request(struct rpc_rqst *req) 953a246b010SChuck Lever { 954a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 955ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 956262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 957f279cd00SJason Baron int sent = 0; 958262965f5SChuck Lever int status; 959262965f5SChuck Lever 960262965f5SChuck Lever xs_pktdump("packet data:", 961262965f5SChuck Lever req->rq_svec->iov_base, 962262965f5SChuck Lever req->rq_svec->iov_len); 963262965f5SChuck Lever 96401d37c42STrond Myklebust if (!xprt_bound(xprt)) 96501d37c42STrond Myklebust return -ENOTCONN; 96675891f50STrond Myklebust 96775891f50STrond Myklebust if (!xprt_request_get_cong(xprt, req)) 96875891f50STrond Myklebust return -EBADSLT; 96975891f50STrond Myklebust 97078215759SChuck Lever req->rq_xtime = ktime_get(); 971f279cd00SJason Baron status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen, 9726c7a64e5STrond Myklebust xdr, 0, true, &sent); 973262965f5SChuck Lever 974262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n", 9756c7a64e5STrond Myklebust xdr->len, status); 976262965f5SChuck Lever 9773dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */ 9783dedbb5cSJason Baron if (status == -EPERM) 9793dedbb5cSJason Baron goto process_status; 9803dedbb5cSJason Baron 981743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 982743c69e7SNeilBrown status = -ENOBUFS; 983743c69e7SNeilBrown 984f279cd00SJason Baron if (sent > 0 || status == 0) { 985f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 986f279cd00SJason Baron if (sent >= req->rq_slen) 987262965f5SChuck Lever return 0; 988262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */ 989262965f5SChuck Lever status = -EAGAIN; 9902199700fSTrond Myklebust } 991262965f5SChuck Lever 9923dedbb5cSJason Baron process_status: 993262965f5SChuck Lever switch (status) { 994fba91afbSTrond Myklebust case -ENOTSOCK: 995fba91afbSTrond Myklebust status = -ENOTCONN; 996fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 997fba91afbSTrond Myklebust break; 998b6ddf64fSTrond Myklebust case -EAGAIN: 999c544577dSTrond Myklebust status = xs_nospace(req); 1000b6ddf64fSTrond Myklebust break; 1001262965f5SChuck Lever case -ENETUNREACH: 10023601c4a9STrond Myklebust case -ENOBUFS: 1003262965f5SChuck Lever case -EPIPE: 1004262965f5SChuck Lever case -ECONNREFUSED: 10053dedbb5cSJason Baron case -EPERM: 1006262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message 1007262965f5SChuck Lever * prompts ECONNREFUSED. */ 100813331a55STrond Myklebust break; 100913331a55STrond Myklebust default: 101013331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 101113331a55STrond Myklebust -status); 1012262965f5SChuck Lever } 10135fe46e9dSBian Naimeng 1014262965f5SChuck Lever return status; 1015262965f5SChuck Lever } 1016262965f5SChuck Lever 1017e06799f9STrond Myklebust /** 1018262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket 101950f484e2STrond Myklebust * @req: pointer to RPC request 1020262965f5SChuck Lever * 1021262965f5SChuck Lever * Return values: 1022262965f5SChuck Lever * 0: The request has been sent 1023262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to 1024262965f5SChuck Lever * complete the request 1025262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 102625985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 1027262965f5SChuck Lever * 1028262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up 1029262965f5SChuck Lever * if sendmsg is not able to make progress? 1030262965f5SChuck Lever */ 1031adfa7144STrond Myklebust static int xs_tcp_send_request(struct rpc_rqst *req) 1032262965f5SChuck Lever { 1033262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 1034ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1035262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 1036a6b31d18STrond Myklebust bool zerocopy = true; 10379ffadfbcSTrond Myklebust bool vm_wait = false; 1038b595bb15SChuck Lever int status; 1039f279cd00SJason Baron int sent; 1040a246b010SChuck Lever 10414cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 10424cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 10434cd34e7cSTrond Myklebust if (transport->sock != NULL) 10444cd34e7cSTrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR); 10454cd34e7cSTrond Myklebust return -ENOTCONN; 10464cd34e7cSTrond Myklebust } 10474cd34e7cSTrond Myklebust 104861677eeeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 1049262965f5SChuck Lever 1050262965f5SChuck Lever xs_pktdump("packet data:", 1051262965f5SChuck Lever req->rq_svec->iov_base, 1052262965f5SChuck Lever req->rq_svec->iov_len); 1053a6b31d18STrond Myklebust /* Don't use zero copy if this is a resend. If the RPC call 1054a6b31d18STrond Myklebust * completes while the socket holds a reference to the pages, 1055a6b31d18STrond Myklebust * then we may end up resending corrupted data. 1056a6b31d18STrond Myklebust */ 105750f484e2STrond Myklebust if (req->rq_task->tk_flags & RPC_TASK_SENT) 1058a6b31d18STrond Myklebust zerocopy = false; 1059a246b010SChuck Lever 10607196dbb0STrond Myklebust if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state)) 10617196dbb0STrond Myklebust xs_tcp_set_socket_timeouts(xprt, transport->sock); 10627196dbb0STrond Myklebust 1063a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful 1064a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have 1065262965f5SChuck Lever * called sendmsg(). */ 106678215759SChuck Lever req->rq_xtime = ktime_get(); 1067a246b010SChuck Lever while (1) { 1068f279cd00SJason Baron sent = 0; 1069f279cd00SJason Baron status = xs_sendpages(transport->sock, NULL, 0, xdr, 10706c7a64e5STrond Myklebust transport->xmit.offset, 10716c7a64e5STrond Myklebust zerocopy, &sent); 1072a246b010SChuck Lever 1073262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n", 10746c7a64e5STrond Myklebust xdr->len - transport->xmit.offset, status); 1075262965f5SChuck Lever 1076a246b010SChuck Lever /* If we've sent the entire packet, immediately 1077a246b010SChuck Lever * reset the count of bytes sent. */ 10786c7a64e5STrond Myklebust transport->xmit.offset += sent; 10796c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 1080262965f5SChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 10816c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 1082a246b010SChuck Lever req->rq_bytes_sent = 0; 10836c7a64e5STrond Myklebust transport->xmit.offset = 0; 1084a246b010SChuck Lever return 0; 1085a246b010SChuck Lever } 1086262965f5SChuck Lever 10879ffadfbcSTrond Myklebust WARN_ON_ONCE(sent == 0 && status == 0); 10889ffadfbcSTrond Myklebust 10899ffadfbcSTrond Myklebust if (status == -EAGAIN ) { 10909ffadfbcSTrond Myklebust /* 10919ffadfbcSTrond Myklebust * Return EAGAIN if we're sure we're hitting the 10929ffadfbcSTrond Myklebust * socket send buffer limits. 10939ffadfbcSTrond Myklebust */ 10949ffadfbcSTrond Myklebust if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) 10959ffadfbcSTrond Myklebust break; 10969ffadfbcSTrond Myklebust /* 10979ffadfbcSTrond Myklebust * Did we hit a memory allocation failure? 10989ffadfbcSTrond Myklebust */ 10999ffadfbcSTrond Myklebust if (sent == 0) { 11009ffadfbcSTrond Myklebust status = -ENOBUFS; 11019ffadfbcSTrond Myklebust if (vm_wait) 11029ffadfbcSTrond Myklebust break; 11039ffadfbcSTrond Myklebust /* Retry, knowing now that we're below the 11049ffadfbcSTrond Myklebust * socket send buffer limit 11059ffadfbcSTrond Myklebust */ 11069ffadfbcSTrond Myklebust vm_wait = true; 11079ffadfbcSTrond Myklebust } 11089ffadfbcSTrond Myklebust continue; 11099ffadfbcSTrond Myklebust } 1110f580dd04STrond Myklebust if (status < 0) 1111f580dd04STrond Myklebust break; 11129ffadfbcSTrond Myklebust vm_wait = false; 1113a246b010SChuck Lever } 1114a246b010SChuck Lever 1115262965f5SChuck Lever switch (status) { 1116fba91afbSTrond Myklebust case -ENOTSOCK: 1117fba91afbSTrond Myklebust status = -ENOTCONN; 1118fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 1119fba91afbSTrond Myklebust break; 1120262965f5SChuck Lever case -EAGAIN: 1121c544577dSTrond Myklebust status = xs_nospace(req); 1122262965f5SChuck Lever break; 1123262965f5SChuck Lever case -ECONNRESET: 1124262965f5SChuck Lever case -ECONNREFUSED: 1125262965f5SChuck Lever case -ENOTCONN: 11263913c78cSTrond Myklebust case -EADDRINUSE: 1127b5872f0cSTrond Myklebust case -ENOBUFS: 1128b9d2bb2eSTrond Myklebust case -EPIPE: 112913331a55STrond Myklebust break; 113013331a55STrond Myklebust default: 113113331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 113213331a55STrond Myklebust -status); 1133a246b010SChuck Lever } 11345fe46e9dSBian Naimeng 1135a246b010SChuck Lever return status; 1136a246b010SChuck Lever } 1137a246b010SChuck Lever 11382a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) 11392a9e1cfaSTrond Myklebust { 11402a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready; 11412a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change; 11422a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space; 11432118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report; 11442a9e1cfaSTrond Myklebust } 11452a9e1cfaSTrond Myklebust 11462a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) 11472a9e1cfaSTrond Myklebust { 11482a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready; 11492a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change; 11502a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space; 11512118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report; 11522118071dSTrond Myklebust } 11532118071dSTrond Myklebust 115442d42a5bSTrond Myklebust static void xs_sock_reset_state_flags(struct rpc_xprt *xprt) 115542d42a5bSTrond Myklebust { 115642d42a5bSTrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 115742d42a5bSTrond Myklebust 115842d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 115942d42a5bSTrond Myklebust } 116042d42a5bSTrond Myklebust 1161b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt) 1162b70ae915STrond Myklebust { 1163b70ae915STrond Myklebust smp_mb__before_atomic(); 1164b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 1165b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 116642d42a5bSTrond Myklebust xs_sock_reset_state_flags(xprt); 1167b70ae915STrond Myklebust smp_mb__after_atomic(); 1168b70ae915STrond Myklebust } 1169b70ae915STrond Myklebust 11702118071dSTrond Myklebust /** 11712118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors 11722118071dSTrond Myklebust * @sk: socket 11732118071dSTrond Myklebust * 11742118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task 11752118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err. 11762118071dSTrond Myklebust */ 11772118071dSTrond Myklebust static void xs_error_report(struct sock *sk) 11782118071dSTrond Myklebust { 11792118071dSTrond Myklebust struct rpc_xprt *xprt; 11802118071dSTrond Myklebust int err; 11812118071dSTrond Myklebust 11822118071dSTrond Myklebust read_lock_bh(&sk->sk_callback_lock); 11832118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 11842118071dSTrond Myklebust goto out; 11852118071dSTrond Myklebust 11862118071dSTrond Myklebust err = -sk->sk_err; 11872118071dSTrond Myklebust if (err == 0) 11882118071dSTrond Myklebust goto out; 11892118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n", 11902118071dSTrond Myklebust xprt, -err); 1191e8353c76STrond Myklebust trace_rpc_socket_error(xprt, sk->sk_socket, err); 11922118071dSTrond Myklebust xprt_wake_pending_tasks(xprt, err); 11932118071dSTrond Myklebust out: 11942118071dSTrond Myklebust read_unlock_bh(&sk->sk_callback_lock); 11952a9e1cfaSTrond Myklebust } 11962a9e1cfaSTrond Myklebust 1197fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport) 1198a246b010SChuck Lever { 1199ee0ac0c2SChuck Lever struct socket *sock = transport->sock; 1200ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 12016cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 1202a246b010SChuck Lever 1203fe315e76SChuck Lever if (sk == NULL) 1204fe315e76SChuck Lever return; 12059903cd1cSChuck Lever 1206264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper)) 1207264d1df3SJeff Layton sk_clear_memalloc(sk); 1208264d1df3SJeff Layton 120909939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 121009939204STrond Myklebust 1211edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 1212a246b010SChuck Lever write_lock_bh(&sk->sk_callback_lock); 1213ee0ac0c2SChuck Lever transport->inet = NULL; 1214ee0ac0c2SChuck Lever transport->sock = NULL; 1215a246b010SChuck Lever 1216a246b010SChuck Lever sk->sk_user_data = NULL; 12172a9e1cfaSTrond Myklebust 12182a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk); 12190c78789eSTrond Myklebust xprt_clear_connected(xprt); 1220a246b010SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 12216cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt); 1222edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 1223a246b010SChuck Lever 12246cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock); 1225a246b010SChuck Lever sock_release(sock); 1226fe315e76SChuck Lever } 1227fe315e76SChuck Lever 1228fe315e76SChuck Lever /** 1229fe315e76SChuck Lever * xs_close - close a socket 1230fe315e76SChuck Lever * @xprt: transport 1231fe315e76SChuck Lever * 1232fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains 1233fe315e76SChuck Lever * on the server we want to save. 1234f75e6745STrond Myklebust * 1235f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with 1236f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer. 1237fe315e76SChuck Lever */ 1238fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt) 1239fe315e76SChuck Lever { 1240fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1241fe315e76SChuck Lever 1242fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt); 1243fe315e76SChuck Lever 1244fe315e76SChuck Lever xs_reset_transport(transport); 124561d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 1246fe315e76SChuck Lever 124762da3b24STrond Myklebust xprt_disconnect_done(xprt); 1248a246b010SChuck Lever } 1249a246b010SChuck Lever 12504a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt) 12514a068258SChuck Lever { 12524a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n", 12534a068258SChuck Lever xprt); 12544a068258SChuck Lever xprt_disconnect_done(xprt); 12554a068258SChuck Lever } 12564a068258SChuck Lever 1257315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt) 1258315f3812SKinglong Mee { 1259315f3812SKinglong Mee xs_free_peer_addresses(xprt); 1260315f3812SKinglong Mee xprt_free(xprt); 1261315f3812SKinglong Mee } 1262315f3812SKinglong Mee 12639903cd1cSChuck Lever /** 12649903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport 12659903cd1cSChuck Lever * @xprt: doomed transport 12669903cd1cSChuck Lever * 12679903cd1cSChuck Lever */ 12689903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt) 1269a246b010SChuck Lever { 127003c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt, 127103c78827STrond Myklebust struct sock_xprt, xprt); 12729903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt); 12739903cd1cSChuck Lever 127403c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker); 1275a1311d87STrond Myklebust xs_close(xprt); 1276edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker); 1277315f3812SKinglong Mee xs_xprt_free(xprt); 1278a1311d87STrond Myklebust module_put(THIS_MODULE); 1279a246b010SChuck Lever } 1280a246b010SChuck Lever 12819903cd1cSChuck Lever /** 1282f9b2ee71STrond Myklebust * xs_udp_data_read_skb - receive callback for UDP sockets 1283f9b2ee71STrond Myklebust * @xprt: transport 1284f9b2ee71STrond Myklebust * @sk: socket 1285f9b2ee71STrond Myklebust * @skb: skbuff 12869903cd1cSChuck Lever * 1287a246b010SChuck Lever */ 1288f9b2ee71STrond Myklebust static void xs_udp_data_read_skb(struct rpc_xprt *xprt, 1289f9b2ee71STrond Myklebust struct sock *sk, 1290f9b2ee71STrond Myklebust struct sk_buff *skb) 1291a246b010SChuck Lever { 1292a246b010SChuck Lever struct rpc_task *task; 1293a246b010SChuck Lever struct rpc_rqst *rovr; 1294f9b2ee71STrond Myklebust int repsize, copied; 1295d8ed029dSAlexey Dobriyan u32 _xid; 1296d8ed029dSAlexey Dobriyan __be32 *xp; 1297a246b010SChuck Lever 12981da8c681SWillem de Bruijn repsize = skb->len; 1299a246b010SChuck Lever if (repsize < 4) { 13009903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize); 1301f9b2ee71STrond Myklebust return; 1302a246b010SChuck Lever } 1303a246b010SChuck Lever 1304a246b010SChuck Lever /* Copy the XID from the skb... */ 13051da8c681SWillem de Bruijn xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid); 1306a246b010SChuck Lever if (xp == NULL) 1307f9b2ee71STrond Myklebust return; 1308a246b010SChuck Lever 1309a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */ 131075c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1311a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 1312a246b010SChuck Lever if (!rovr) 1313a246b010SChuck Lever goto out_unlock; 1314729749bbSTrond Myklebust xprt_pin_rqst(rovr); 1315ecd465eeSChuck Lever xprt_update_rtt(rovr->rq_task); 131675c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1317a246b010SChuck Lever task = rovr->rq_task; 1318a246b010SChuck Lever 1319a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize) 1320a246b010SChuck Lever copied = repsize; 1321a246b010SChuck Lever 1322a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */ 13231781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) { 132475c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13250afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INERRORS); 1326729749bbSTrond Myklebust goto out_unpin; 13271781f7f5SHerbert Xu } 13281781f7f5SHerbert Xu 1329a246b010SChuck Lever 1330729749bbSTrond Myklebust spin_lock_bh(&xprt->transport_lock); 13316a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied); 1332ce7c252aSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 133375c84151STrond Myklebust spin_lock(&xprt->queue_lock); 13341570c1e4SChuck Lever xprt_complete_rqst(task, copied); 13350afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS); 1336729749bbSTrond Myklebust out_unpin: 1337729749bbSTrond Myklebust xprt_unpin_rqst(rovr); 1338a246b010SChuck Lever out_unlock: 133975c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1340f9b2ee71STrond Myklebust } 1341f9b2ee71STrond Myklebust 1342f9b2ee71STrond Myklebust static void xs_udp_data_receive(struct sock_xprt *transport) 1343f9b2ee71STrond Myklebust { 1344f9b2ee71STrond Myklebust struct sk_buff *skb; 1345f9b2ee71STrond Myklebust struct sock *sk; 1346f9b2ee71STrond Myklebust int err; 1347f9b2ee71STrond Myklebust 1348f9b2ee71STrond Myklebust mutex_lock(&transport->recv_mutex); 1349f9b2ee71STrond Myklebust sk = transport->inet; 1350f9b2ee71STrond Myklebust if (sk == NULL) 1351f9b2ee71STrond Myklebust goto out; 13524f546149STrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 1353f9b2ee71STrond Myklebust for (;;) { 13547c13f97fSPaolo Abeni skb = skb_recv_udp(sk, 0, 1, &err); 13554f546149STrond Myklebust if (skb == NULL) 13564f546149STrond Myklebust break; 1357f9b2ee71STrond Myklebust xs_udp_data_read_skb(&transport->xprt, sk, skb); 1358850cbaddSPaolo Abeni consume_skb(skb); 13590af3442aSTrond Myklebust cond_resched(); 1360f9b2ee71STrond Myklebust } 1361a246b010SChuck Lever out: 1362f9b2ee71STrond Myklebust mutex_unlock(&transport->recv_mutex); 1363f9b2ee71STrond Myklebust } 1364f9b2ee71STrond Myklebust 1365f9b2ee71STrond Myklebust static void xs_udp_data_receive_workfn(struct work_struct *work) 1366f9b2ee71STrond Myklebust { 1367f9b2ee71STrond Myklebust struct sock_xprt *transport = 1368f9b2ee71STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1369f9b2ee71STrond Myklebust xs_udp_data_receive(transport); 1370f9b2ee71STrond Myklebust } 1371f9b2ee71STrond Myklebust 1372f9b2ee71STrond Myklebust /** 1373f9b2ee71STrond Myklebust * xs_data_ready - "data ready" callback for UDP sockets 1374f9b2ee71STrond Myklebust * @sk: socket with data to read 1375f9b2ee71STrond Myklebust * 1376f9b2ee71STrond Myklebust */ 1377f9b2ee71STrond Myklebust static void xs_data_ready(struct sock *sk) 1378f9b2ee71STrond Myklebust { 1379f9b2ee71STrond Myklebust struct rpc_xprt *xprt; 1380f9b2ee71STrond Myklebust 1381f9b2ee71STrond Myklebust read_lock_bh(&sk->sk_callback_lock); 1382f9b2ee71STrond Myklebust dprintk("RPC: xs_data_ready...\n"); 1383f9b2ee71STrond Myklebust xprt = xprt_from_sock(sk); 1384f9b2ee71STrond Myklebust if (xprt != NULL) { 1385f9b2ee71STrond Myklebust struct sock_xprt *transport = container_of(xprt, 1386f9b2ee71STrond Myklebust struct sock_xprt, xprt); 13875157b956STrond Myklebust transport->old_data_ready(sk); 13885157b956STrond Myklebust /* Any data means we had a useful conversation, so 13895157b956STrond Myklebust * then we don't need to delay the next reconnect 13905157b956STrond Myklebust */ 13915157b956STrond Myklebust if (xprt->reestablish_timeout) 13925157b956STrond Myklebust xprt->reestablish_timeout = 0; 139342d42a5bSTrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 139440a5f1b1STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 1395f9b2ee71STrond Myklebust } 1396f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1397a246b010SChuck Lever } 1398a246b010SChuck Lever 1399a519fc7aSTrond Myklebust /* 1400a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending 1401a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT 1402a519fc7aSTrond Myklebust */ 1403a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt) 1404a519fc7aSTrond Myklebust { 1405a519fc7aSTrond Myklebust xprt_force_disconnect(xprt); 1406a519fc7aSTrond Myklebust } 1407a519fc7aSTrond Myklebust 14089e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 140976566773SChuck Lever static int xs_tcp_bc_up(struct svc_serv *serv, struct net *net) 141076566773SChuck Lever { 141176566773SChuck Lever int ret; 141276566773SChuck Lever 141376566773SChuck Lever ret = svc_create_xprt(serv, "tcp-bc", net, PF_INET, 0, 141476566773SChuck Lever SVC_SOCK_ANONYMOUS); 141576566773SChuck Lever if (ret < 0) 141676566773SChuck Lever return ret; 141776566773SChuck Lever return 0; 141876566773SChuck Lever } 14196b26cc8cSChuck Lever 14206b26cc8cSChuck Lever static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt) 14216b26cc8cSChuck Lever { 14226b26cc8cSChuck Lever return PAGE_SIZE; 14236b26cc8cSChuck Lever } 14249e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 142544b98efdSRicardo Labiaga 14269903cd1cSChuck Lever /** 14279903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes 14289903cd1cSChuck Lever * @sk: socket whose state has changed 14299903cd1cSChuck Lever * 14309903cd1cSChuck Lever */ 14319903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk) 1432a246b010SChuck Lever { 1433a246b010SChuck Lever struct rpc_xprt *xprt; 14340fdea1e8STrond Myklebust struct sock_xprt *transport; 1435a246b010SChuck Lever 1436f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1437a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk))) 1438a246b010SChuck Lever goto out; 14399903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt); 1440669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n", 1441a246b010SChuck Lever sk->sk_state, xprt_connected(xprt), 1442a246b010SChuck Lever sock_flag(sk, SOCK_DEAD), 1443669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED), 1444669502ffSAndy Chittenden sk->sk_shutdown); 1445a246b010SChuck Lever 14460fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 144740b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket); 1448a246b010SChuck Lever switch (sk->sk_state) { 1449a246b010SChuck Lever case TCP_ESTABLISHED: 1450f064af1eSEric Dumazet spin_lock(&xprt->transport_lock); 1451a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) { 14528b71798cSTrond Myklebust xprt->connect_cookie++; 14530fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 14540fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 145551971139SChuck Lever 14563968a8a5SChuck Lever xprt->stat.connect_count++; 14573968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 14583968a8a5SChuck Lever xprt->stat.connect_start; 14592a491991STrond Myklebust xprt_wake_pending_tasks(xprt, -EAGAIN); 1460a246b010SChuck Lever } 1461f064af1eSEric Dumazet spin_unlock(&xprt->transport_lock); 1462a246b010SChuck Lever break; 14633b948ae5STrond Myklebust case TCP_FIN_WAIT1: 14643b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */ 14657c1d71cfSTrond Myklebust xprt->connect_cookie++; 1466663b8858STrond Myklebust xprt->reestablish_timeout = 0; 14673b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14684e857c58SPeter Zijlstra smp_mb__before_atomic(); 14693b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1470ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 14714e857c58SPeter Zijlstra smp_mb__after_atomic(); 1472a246b010SChuck Lever break; 1473632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT: 14743b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */ 14757c1d71cfSTrond Myklebust xprt->connect_cookie++; 1476d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1477a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 1478e9d47639SGustavo A. R. Silva /* fall through */ 1479663b8858STrond Myklebust case TCP_CLOSING: 1480663b8858STrond Myklebust /* 1481663b8858STrond Myklebust * If the server closed down the connection, make sure that 1482663b8858STrond Myklebust * we back off before reconnecting 1483663b8858STrond Myklebust */ 1484663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 1485663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 14863b948ae5STrond Myklebust break; 14873b948ae5STrond Myklebust case TCP_LAST_ACK: 1488670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 14894e857c58SPeter Zijlstra smp_mb__before_atomic(); 14903b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 14914e857c58SPeter Zijlstra smp_mb__after_atomic(); 14923b948ae5STrond Myklebust break; 14933b948ae5STrond Myklebust case TCP_CLOSE: 14940fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING, 14950fdea1e8STrond Myklebust &transport->sock_state)) 14960fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 14979b30889cSTrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 14983ffbc1d6SNeilBrown if (sk->sk_err) 14993ffbc1d6SNeilBrown xprt_wake_pending_tasks(xprt, -sk->sk_err); 15009b30889cSTrond Myklebust /* Trigger the socket release */ 15019b30889cSTrond Myklebust xs_tcp_force_close(xprt); 1502a246b010SChuck Lever } 1503a246b010SChuck Lever out: 1504f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1505a246b010SChuck Lever } 1506a246b010SChuck Lever 15071f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk) 15081f0fa154SIlpo Järvinen { 150913331a55STrond Myklebust struct socket_wq *wq; 15101f0fa154SIlpo Järvinen struct rpc_xprt *xprt; 15111f0fa154SIlpo Järvinen 151213331a55STrond Myklebust if (!sk->sk_socket) 15131f0fa154SIlpo Järvinen return; 151413331a55STrond Myklebust clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 15151f0fa154SIlpo Järvinen 15161f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk)))) 15171f0fa154SIlpo Järvinen return; 151813331a55STrond Myklebust rcu_read_lock(); 151913331a55STrond Myklebust wq = rcu_dereference(sk->sk_wq); 152013331a55STrond Myklebust if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0) 152113331a55STrond Myklebust goto out; 15221f0fa154SIlpo Järvinen 1523c544577dSTrond Myklebust if (xprt_write_space(xprt)) 1524c544577dSTrond Myklebust sk->sk_write_pending--; 152513331a55STrond Myklebust out: 152613331a55STrond Myklebust rcu_read_unlock(); 15271f0fa154SIlpo Järvinen } 15281f0fa154SIlpo Järvinen 15292a9e1cfaSTrond Myklebust /** 1530c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space 1531c7b2cae8SChuck Lever * becomes available 15329903cd1cSChuck Lever * @sk: socket whose state has changed 15339903cd1cSChuck Lever * 1534a246b010SChuck Lever * Called when more output buffer space is available for this socket. 1535a246b010SChuck Lever * We try not to wake our writers until they can make "significant" 1536c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1537a246b010SChuck Lever * with a bunch of small requests. 1538a246b010SChuck Lever */ 1539c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk) 1540a246b010SChuck Lever { 1541f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1542c7b2cae8SChuck Lever 1543c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */ 15441f0fa154SIlpo Järvinen if (sock_writeable(sk)) 15451f0fa154SIlpo Järvinen xs_write_space(sk); 1546c7b2cae8SChuck Lever 1547f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1548c7b2cae8SChuck Lever } 1549c7b2cae8SChuck Lever 1550c7b2cae8SChuck Lever /** 1551c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space 1552c7b2cae8SChuck Lever * becomes available 1553c7b2cae8SChuck Lever * @sk: socket whose state has changed 1554c7b2cae8SChuck Lever * 1555c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket. 1556c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant" 1557c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1558c7b2cae8SChuck Lever * with a bunch of small requests. 1559c7b2cae8SChuck Lever */ 1560c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk) 1561c7b2cae8SChuck Lever { 1562f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1563c7b2cae8SChuck Lever 1564c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */ 156564dc6130SEric Dumazet if (sk_stream_is_writeable(sk)) 15661f0fa154SIlpo Järvinen xs_write_space(sk); 1567c7b2cae8SChuck Lever 1568f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1569a246b010SChuck Lever } 1570a246b010SChuck Lever 1571470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) 1572a246b010SChuck Lever { 1573ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1574ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 1575a246b010SChuck Lever 15767c6e066eSChuck Lever if (transport->rcvsize) { 1577a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 15787c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2; 1579a246b010SChuck Lever } 15807c6e066eSChuck Lever if (transport->sndsize) { 1581a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 15827c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; 1583a246b010SChuck Lever sk->sk_write_space(sk); 1584a246b010SChuck Lever } 1585a246b010SChuck Lever } 1586a246b010SChuck Lever 158743118c29SChuck Lever /** 1588470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits 158943118c29SChuck Lever * @xprt: generic transport 1590470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes 1591470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes 159243118c29SChuck Lever * 1593470056c2SChuck Lever * Set socket send and receive buffer size limits. 159443118c29SChuck Lever */ 1595470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize) 159643118c29SChuck Lever { 15977c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 15987c6e066eSChuck Lever 15997c6e066eSChuck Lever transport->sndsize = 0; 1600470056c2SChuck Lever if (sndsize) 16017c6e066eSChuck Lever transport->sndsize = sndsize + 1024; 16027c6e066eSChuck Lever transport->rcvsize = 0; 1603470056c2SChuck Lever if (rcvsize) 16047c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024; 1605470056c2SChuck Lever 1606470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 160743118c29SChuck Lever } 160843118c29SChuck Lever 160946c0ee8bSChuck Lever /** 161046c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport 161146c0ee8bSChuck Lever * @task: task that timed out 161246c0ee8bSChuck Lever * 161346c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred. 161446c0ee8bSChuck Lever */ 16156a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task) 161646c0ee8bSChuck Lever { 1617b977b644SChuck Lever spin_lock_bh(&xprt->transport_lock); 16186a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT); 1619b977b644SChuck Lever spin_unlock_bh(&xprt->transport_lock); 162046c0ee8bSChuck Lever } 162146c0ee8bSChuck Lever 1622826799e6SJ. Bruce Fields static int xs_get_random_port(void) 1623b85d8806SChuck Lever { 1624826799e6SJ. Bruce Fields unsigned short min = xprt_min_resvport, max = xprt_max_resvport; 1625826799e6SJ. Bruce Fields unsigned short range; 1626826799e6SJ. Bruce Fields unsigned short rand; 1627826799e6SJ. Bruce Fields 1628826799e6SJ. Bruce Fields if (max < min) 1629826799e6SJ. Bruce Fields return -EADDRINUSE; 1630826799e6SJ. Bruce Fields range = max - min + 1; 1631826799e6SJ. Bruce Fields rand = (unsigned short) prandom_u32() % range; 1632826799e6SJ. Bruce Fields return rand + min; 1633b85d8806SChuck Lever } 1634b85d8806SChuck Lever 163592200412SChuck Lever /** 16364dda9c8aSTrond Myklebust * xs_set_reuseaddr_port - set the socket's port and address reuse options 16374dda9c8aSTrond Myklebust * @sock: socket 16384dda9c8aSTrond Myklebust * 16394dda9c8aSTrond Myklebust * Note that this function has to be called on all sockets that share the 16404dda9c8aSTrond Myklebust * same port, and it must be called before binding. 16414dda9c8aSTrond Myklebust */ 16424dda9c8aSTrond Myklebust static void xs_sock_set_reuseport(struct socket *sock) 16434dda9c8aSTrond Myklebust { 1644402e23b4STrond Myklebust int opt = 1; 16454dda9c8aSTrond Myklebust 1646402e23b4STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, 1647402e23b4STrond Myklebust (char *)&opt, sizeof(opt)); 16484dda9c8aSTrond Myklebust } 16494dda9c8aSTrond Myklebust 16504dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock) 16514dda9c8aSTrond Myklebust { 16524dda9c8aSTrond Myklebust struct sockaddr_storage buf; 16534dda9c8aSTrond Myklebust unsigned short port = 0; 16544dda9c8aSTrond Myklebust 16559b2c45d4SDenys Vlasenko if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0) 16564dda9c8aSTrond Myklebust goto out; 16574dda9c8aSTrond Myklebust switch (buf.ss_family) { 16584dda9c8aSTrond Myklebust case AF_INET6: 16594dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port); 16604dda9c8aSTrond Myklebust break; 16614dda9c8aSTrond Myklebust case AF_INET: 16624dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port); 16634dda9c8aSTrond Myklebust } 16644dda9c8aSTrond Myklebust out: 16654dda9c8aSTrond Myklebust return port; 16664dda9c8aSTrond Myklebust } 16674dda9c8aSTrond Myklebust 16684dda9c8aSTrond Myklebust /** 166992200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address 167092200412SChuck Lever * @xprt: generic transport 167192200412SChuck Lever * @port: new port number 167292200412SChuck Lever * 167392200412SChuck Lever */ 167492200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) 167592200412SChuck Lever { 167692200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port); 1677c4efcb1dSChuck Lever 16789dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port); 16799dc3b095SChuck Lever xs_update_peer_port(xprt); 168092200412SChuck Lever } 168192200412SChuck Lever 16824dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock) 16834dda9c8aSTrond Myklebust { 16844dda9c8aSTrond Myklebust if (transport->srcport == 0) 16854dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock); 16864dda9c8aSTrond Myklebust } 16874dda9c8aSTrond Myklebust 1688826799e6SJ. Bruce Fields static int xs_get_srcport(struct sock_xprt *transport) 168967a391d7STrond Myklebust { 1690826799e6SJ. Bruce Fields int port = transport->srcport; 169167a391d7STrond Myklebust 169267a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport) 169367a391d7STrond Myklebust port = xs_get_random_port(); 169467a391d7STrond Myklebust return port; 169567a391d7STrond Myklebust } 169667a391d7STrond Myklebust 1697baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) 169867a391d7STrond Myklebust { 1699fbfffbd5SChuck Lever if (transport->srcport != 0) 1700fbfffbd5SChuck Lever transport->srcport = 0; 170167a391d7STrond Myklebust if (!transport->xprt.resvport) 170267a391d7STrond Myklebust return 0; 170367a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport) 170467a391d7STrond Myklebust return xprt_max_resvport; 170567a391d7STrond Myklebust return --port; 170667a391d7STrond Myklebust } 1707beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock) 1708a246b010SChuck Lever { 1709beb59b68SPavel Emelyanov struct sockaddr_storage myaddr; 171067a391d7STrond Myklebust int err, nloop = 0; 1711826799e6SJ. Bruce Fields int port = xs_get_srcport(transport); 171267a391d7STrond Myklebust unsigned short last; 1713a246b010SChuck Lever 17140f7a622cSChris Perl /* 17150f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 && 17160f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local 17170f7a622cSChris Perl * port selection happen implicitly when the socket is used 17180f7a622cSChris Perl * (for example at connect time). 17190f7a622cSChris Perl * 17200f7a622cSChris Perl * This ensures that we can continue to establish TCP 17210f7a622cSChris Perl * connections even when all local ephemeral ports are already 17220f7a622cSChris Perl * a part of some TCP connection. This makes no difference 17230f7a622cSChris Perl * for UDP sockets, but also doens't harm them. 17240f7a622cSChris Perl * 17250f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 && 17260f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will 17270f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed. 17280f7a622cSChris Perl */ 1729826799e6SJ. Bruce Fields if (port <= 0) 1730826799e6SJ. Bruce Fields return port; 17310f7a622cSChris Perl 1732beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen); 1733a246b010SChuck Lever do { 1734beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port); 1735e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr, 1736beb59b68SPavel Emelyanov transport->xprt.addrlen); 1737a246b010SChuck Lever if (err == 0) { 1738fbfffbd5SChuck Lever transport->srcport = port; 1739d3bc9a1dSFrank van Maarseveen break; 1740a246b010SChuck Lever } 174167a391d7STrond Myklebust last = port; 1742baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port); 174367a391d7STrond Myklebust if (port > last) 174467a391d7STrond Myklebust nloop++; 174567a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2); 1746beb59b68SPavel Emelyanov 17474232e863SChuck Lever if (myaddr.ss_family == AF_INET) 1748beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__, 1749beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr, 1750beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err); 1751beb59b68SPavel Emelyanov else 1752beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__, 1753beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr, 17547dc753f0SChuck Lever port, err ? "failed" : "ok", err); 1755a246b010SChuck Lever return err; 1756a246b010SChuck Lever } 1757a246b010SChuck Lever 1758176e21eeSChuck Lever /* 1759176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets 1760176e21eeSChuck Lever */ 1761176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task) 1762176e21eeSChuck Lever { 1763fb43d172STrond Myklebust xprt_set_bound(task->tk_xprt); 1764176e21eeSChuck Lever } 1765176e21eeSChuck Lever 1766176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) 1767176e21eeSChuck Lever { 1768176e21eeSChuck Lever } 1769a246b010SChuck Lever 1770ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC 1771ed07536eSPeter Zijlstra static struct lock_class_key xs_key[2]; 1772ed07536eSPeter Zijlstra static struct lock_class_key xs_slock_key[2]; 1773ed07536eSPeter Zijlstra 1774176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1775176e21eeSChuck Lever { 1776176e21eeSChuck Lever struct sock *sk = sock->sk; 1777176e21eeSChuck Lever 1778176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC", 1779176e21eeSChuck Lever &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]); 1780176e21eeSChuck Lever } 1781176e21eeSChuck Lever 17828945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 1783ed07536eSPeter Zijlstra { 1784ed07536eSPeter Zijlstra struct sock *sk = sock->sk; 17858945ee5eSChuck Lever 17868945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC", 17878945ee5eSChuck Lever &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]); 1788ed07536eSPeter Zijlstra } 17898945ee5eSChuck Lever 17908945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 17918945ee5eSChuck Lever { 17928945ee5eSChuck Lever struct sock *sk = sock->sk; 17938945ee5eSChuck Lever 17948945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC", 17958945ee5eSChuck Lever &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]); 1796ed07536eSPeter Zijlstra } 17976bc9638aSPavel Emelyanov 17986bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 17996bc9638aSPavel Emelyanov { 1800fafc4e1eSHannes Frederic Sowa if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk))) 18011b7a1819SWeston Andros Adamson return; 18021b7a1819SWeston Andros Adamson 18034232e863SChuck Lever switch (family) { 1804176e21eeSChuck Lever case AF_LOCAL: 1805176e21eeSChuck Lever xs_reclassify_socketu(sock); 1806176e21eeSChuck Lever break; 18074232e863SChuck Lever case AF_INET: 18086bc9638aSPavel Emelyanov xs_reclassify_socket4(sock); 18094232e863SChuck Lever break; 18104232e863SChuck Lever case AF_INET6: 18116bc9638aSPavel Emelyanov xs_reclassify_socket6(sock); 18124232e863SChuck Lever break; 18134232e863SChuck Lever } 18146bc9638aSPavel Emelyanov } 1815ed07536eSPeter Zijlstra #else 18166bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 18176bc9638aSPavel Emelyanov { 18186bc9638aSPavel Emelyanov } 1819ed07536eSPeter Zijlstra #endif 1820ed07536eSPeter Zijlstra 182193dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work) 182293dc41bdSNeilBrown { 182393dc41bdSNeilBrown } 182493dc41bdSNeilBrown 18256bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt, 18264dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type, 18274dda9c8aSTrond Myklebust int protocol, bool reuseport) 182822f79326SPavel Emelyanov { 182922f79326SPavel Emelyanov struct socket *sock; 183022f79326SPavel Emelyanov int err; 183122f79326SPavel Emelyanov 18326bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1); 183322f79326SPavel Emelyanov if (err < 0) { 183422f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n", 183522f79326SPavel Emelyanov protocol, -err); 183622f79326SPavel Emelyanov goto out; 183722f79326SPavel Emelyanov } 18386bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock); 183922f79326SPavel Emelyanov 18404dda9c8aSTrond Myklebust if (reuseport) 18414dda9c8aSTrond Myklebust xs_sock_set_reuseport(sock); 18424dda9c8aSTrond Myklebust 18434cea288aSBen Hutchings err = xs_bind(transport, sock); 18444cea288aSBen Hutchings if (err) { 184522f79326SPavel Emelyanov sock_release(sock); 184622f79326SPavel Emelyanov goto out; 184722f79326SPavel Emelyanov } 184822f79326SPavel Emelyanov 184922f79326SPavel Emelyanov return sock; 185022f79326SPavel Emelyanov out: 185122f79326SPavel Emelyanov return ERR_PTR(err); 185222f79326SPavel Emelyanov } 185322f79326SPavel Emelyanov 1854176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt, 1855176e21eeSChuck Lever struct socket *sock) 1856176e21eeSChuck Lever { 1857176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1858176e21eeSChuck Lever xprt); 1859176e21eeSChuck Lever 1860176e21eeSChuck Lever if (!transport->inet) { 1861176e21eeSChuck Lever struct sock *sk = sock->sk; 1862176e21eeSChuck Lever 1863176e21eeSChuck Lever write_lock_bh(&sk->sk_callback_lock); 1864176e21eeSChuck Lever 1865176e21eeSChuck Lever xs_save_old_callbacks(transport, sk); 1866176e21eeSChuck Lever 1867176e21eeSChuck Lever sk->sk_user_data = xprt; 1868a2648094STrond Myklebust sk->sk_data_ready = xs_data_ready; 1869176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space; 1870b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 18712118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 1872c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 1873176e21eeSChuck Lever 1874176e21eeSChuck Lever xprt_clear_connected(xprt); 1875176e21eeSChuck Lever 1876176e21eeSChuck Lever /* Reset to new socket */ 1877176e21eeSChuck Lever transport->sock = sock; 1878176e21eeSChuck Lever transport->inet = sk; 1879176e21eeSChuck Lever 1880176e21eeSChuck Lever write_unlock_bh(&sk->sk_callback_lock); 1881176e21eeSChuck Lever } 1882176e21eeSChuck Lever 1883550aebfeSTrond Myklebust xs_stream_reset_connect(transport); 18846c7a64e5STrond Myklebust 1885176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0); 1886176e21eeSChuck Lever } 1887176e21eeSChuck Lever 1888176e21eeSChuck Lever /** 1889176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint 1890176e21eeSChuck Lever * @transport: socket transport to connect 1891176e21eeSChuck Lever */ 1892dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport) 1893176e21eeSChuck Lever { 1894176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt; 1895176e21eeSChuck Lever struct socket *sock; 1896176e21eeSChuck Lever int status = -EIO; 1897176e21eeSChuck Lever 1898176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL, 1899176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1); 1900176e21eeSChuck Lever if (status < 0) { 1901176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL " 1902176e21eeSChuck Lever "transport socket (%d).\n", -status); 1903176e21eeSChuck Lever goto out; 1904176e21eeSChuck Lever } 1905d1358917SStefan Hajnoczi xs_reclassify_socket(AF_LOCAL, sock); 1906176e21eeSChuck Lever 1907176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n", 1908176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1909176e21eeSChuck Lever 1910176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock); 191140b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 1912176e21eeSChuck Lever switch (status) { 1913176e21eeSChuck Lever case 0: 1914176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n", 1915176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 19163968a8a5SChuck Lever xprt->stat.connect_count++; 19173968a8a5SChuck Lever xprt->stat.connect_time += (long)jiffies - 19183968a8a5SChuck Lever xprt->stat.connect_start; 1919176e21eeSChuck Lever xprt_set_connected(xprt); 19203601c4a9STrond Myklebust case -ENOBUFS: 1921176e21eeSChuck Lever break; 1922176e21eeSChuck Lever case -ENOENT: 1923176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n", 1924176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 1925176e21eeSChuck Lever break; 19264a20a988STrond Myklebust case -ECONNREFUSED: 19274a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n", 19284a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 19294a20a988STrond Myklebust break; 1930176e21eeSChuck Lever default: 1931176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n", 1932176e21eeSChuck Lever __func__, -status, 1933176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 1934176e21eeSChuck Lever } 1935176e21eeSChuck Lever 1936176e21eeSChuck Lever out: 1937176e21eeSChuck Lever xprt_clear_connecting(xprt); 1938176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status); 1939dc107402SJ. Bruce Fields return status; 1940dc107402SJ. Bruce Fields } 1941dc107402SJ. Bruce Fields 1942b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) 1943dc107402SJ. Bruce Fields { 1944dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1945dc107402SJ. Bruce Fields int ret; 1946dc107402SJ. Bruce Fields 1947dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) { 1948dc107402SJ. Bruce Fields /* 1949dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the 1950dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc 1951dc107402SJ. Bruce Fields * call. Thus we connect synchronously. 1952dc107402SJ. Bruce Fields * 1953dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls, 1954dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to 1955dc107402SJ. Bruce Fields * connect. 1956dc107402SJ. Bruce Fields */ 1957dc107402SJ. Bruce Fields rpc_exit(task, -ENOTCONN); 1958dc107402SJ. Bruce Fields return; 1959dc107402SJ. Bruce Fields } 1960dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport); 1961dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task)) 1962dc107402SJ. Bruce Fields msleep_interruptible(15000); 1963176e21eeSChuck Lever } 1964176e21eeSChuck Lever 19653c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP) 1966d6e971d8SJeff Layton /* 1967d6e971d8SJeff Layton * Note that this should be called with XPRT_LOCKED held (or when we otherwise 1968d6e971d8SJeff Layton * know that we have exclusive access to the socket), to guard against 1969d6e971d8SJeff Layton * races with xs_reset_transport. 1970d6e971d8SJeff Layton */ 1971a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 1972a564b8f0SMel Gorman { 1973a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 1974a564b8f0SMel Gorman xprt); 1975a564b8f0SMel Gorman 1976d6e971d8SJeff Layton /* 1977d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The 1978d6e971d8SJeff Layton * reconnecting process will get it for us. 1979d6e971d8SJeff Layton */ 1980d6e971d8SJeff Layton if (!transport->inet) 1981d6e971d8SJeff Layton return; 19828e228133SJeff Layton if (atomic_read(&xprt->swapper)) 1983a564b8f0SMel Gorman sk_set_memalloc(transport->inet); 1984a564b8f0SMel Gorman } 1985a564b8f0SMel Gorman 1986a564b8f0SMel Gorman /** 1987d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap. 1988a564b8f0SMel Gorman * @xprt: transport to tag 1989a564b8f0SMel Gorman * 19908e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and 19918e228133SJeff Layton * optionally mark it for swapping if it wasn't already. 1992a564b8f0SMel Gorman */ 1993d67fa4d8SJeff Layton static int 1994d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 1995a564b8f0SMel Gorman { 1996d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 1997a564b8f0SMel Gorman 1998d6e971d8SJeff Layton if (atomic_inc_return(&xprt->swapper) != 1) 1999d6e971d8SJeff Layton return 0; 2000d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2001d6e971d8SJeff Layton return -ERESTARTSYS; 2002d6e971d8SJeff Layton if (xs->inet) 2003d6e971d8SJeff Layton sk_set_memalloc(xs->inet); 2004d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 20058e228133SJeff Layton return 0; 2006a564b8f0SMel Gorman } 2007a564b8f0SMel Gorman 20088e228133SJeff Layton /** 2009d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap. 20108e228133SJeff Layton * @xprt: transport to tag 20118e228133SJeff Layton * 20128e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the 20138e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket. 20148e228133SJeff Layton */ 2015d67fa4d8SJeff Layton static void 2016d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 20178e228133SJeff Layton { 2018d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 20198e228133SJeff Layton 2020d6e971d8SJeff Layton if (!atomic_dec_and_test(&xprt->swapper)) 2021d6e971d8SJeff Layton return; 2022d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2023d6e971d8SJeff Layton return; 2024d6e971d8SJeff Layton if (xs->inet) 2025d6e971d8SJeff Layton sk_clear_memalloc(xs->inet); 2026d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 2027a564b8f0SMel Gorman } 2028a564b8f0SMel Gorman #else 2029a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2030a564b8f0SMel Gorman { 2031a564b8f0SMel Gorman } 2032d67fa4d8SJeff Layton 2033d67fa4d8SJeff Layton static int 2034d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2035d67fa4d8SJeff Layton { 2036d67fa4d8SJeff Layton return -EINVAL; 2037d67fa4d8SJeff Layton } 2038d67fa4d8SJeff Layton 2039d67fa4d8SJeff Layton static void 2040d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 2041d67fa4d8SJeff Layton { 2042d67fa4d8SJeff Layton } 2043a564b8f0SMel Gorman #endif 2044a564b8f0SMel Gorman 204516be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2046a246b010SChuck Lever { 204716be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2048edb267a6SChuck Lever 2049ee0ac0c2SChuck Lever if (!transport->inet) { 2050b0d93ad5SChuck Lever struct sock *sk = sock->sk; 2051b0d93ad5SChuck Lever 2052b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2053b0d93ad5SChuck Lever 20542a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 20552a9e1cfaSTrond Myklebust 2056b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2057f9b2ee71STrond Myklebust sk->sk_data_ready = xs_data_ready; 2058b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space; 2059b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 2060c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 2061b0d93ad5SChuck Lever 2062b0d93ad5SChuck Lever xprt_set_connected(xprt); 2063b0d93ad5SChuck Lever 2064b0d93ad5SChuck Lever /* Reset to new socket */ 2065ee0ac0c2SChuck Lever transport->sock = sock; 2066ee0ac0c2SChuck Lever transport->inet = sk; 2067b0d93ad5SChuck Lever 2068a564b8f0SMel Gorman xs_set_memalloc(xprt); 2069a564b8f0SMel Gorman 2070b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2071b0d93ad5SChuck Lever } 2072470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 207302910177STrond Myklebust 207402910177STrond Myklebust xprt->stat.connect_start = jiffies; 207516be2d20SChuck Lever } 207616be2d20SChuck Lever 20778c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work) 2078a246b010SChuck Lever { 2079a246b010SChuck Lever struct sock_xprt *transport = 2080a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work); 2081a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2082d099b8afSColin Ian King struct socket *sock; 2083b65c0310SPavel Emelyanov int status = -EIO; 2084a246b010SChuck Lever 20858c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport, 20864dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM, 20874dda9c8aSTrond Myklebust IPPROTO_UDP, false); 2088b65c0310SPavel Emelyanov if (IS_ERR(sock)) 2089a246b010SChuck Lever goto out; 209068e220bdSChuck Lever 2091c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2092c740eff8SChuck Lever "%s (port %s)\n", xprt, 2093c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2094c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2095c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 209668e220bdSChuck Lever 209768e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock); 209840b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0); 2099a246b010SChuck Lever status = 0; 2100b0d93ad5SChuck Lever out: 2101718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 2102b0d93ad5SChuck Lever xprt_clear_connecting(xprt); 21037d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2104b0d93ad5SChuck Lever } 2105b0d93ad5SChuck Lever 21064876cc77STrond Myklebust /** 21074876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket 21084876cc77STrond Myklebust * @xprt: transport 21094876cc77STrond Myklebust * 21104876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the 21114876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR); 21124876cc77STrond Myklebust */ 21134876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt) 21144876cc77STrond Myklebust { 21154876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21164876cc77STrond Myklebust struct socket *sock = transport->sock; 21179b30889cSTrond Myklebust int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE; 21184876cc77STrond Myklebust 21194876cc77STrond Myklebust if (sock == NULL) 21204876cc77STrond Myklebust return; 21219b30889cSTrond Myklebust switch (skst) { 21229b30889cSTrond Myklebust default: 21234876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 21244876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock); 21259b30889cSTrond Myklebust break; 21269b30889cSTrond Myklebust case TCP_CLOSE: 21279b30889cSTrond Myklebust case TCP_TIME_WAIT: 21284876cc77STrond Myklebust xs_reset_transport(transport); 21294876cc77STrond Myklebust } 21309b30889cSTrond Myklebust } 21314876cc77STrond Myklebust 21328d1b8c62STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 21338d1b8c62STrond Myklebust struct socket *sock) 2134b0d93ad5SChuck Lever { 213516be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21367196dbb0STrond Myklebust unsigned int keepidle; 21377196dbb0STrond Myklebust unsigned int keepcnt; 21387f260e85STrond Myklebust unsigned int opt_on = 1; 2139775f06abSTrond Myklebust unsigned int timeo; 21407f260e85STrond Myklebust 21417196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 21427196dbb0STrond Myklebust keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ); 21437196dbb0STrond Myklebust keepcnt = xprt->timeout->to_retries + 1; 21447196dbb0STrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) * 21457196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 21467196dbb0STrond Myklebust clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 21477196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 21487f260e85STrond Myklebust 21497f260e85STrond Myklebust /* TCP Keepalive options */ 21507f260e85STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, 21517f260e85STrond Myklebust (char *)&opt_on, sizeof(opt_on)); 21527f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, 21537f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21547f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, 21557f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 21567f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, 21577f260e85STrond Myklebust (char *)&keepcnt, sizeof(keepcnt)); 2158b0d93ad5SChuck Lever 21598d1b8c62STrond Myklebust /* TCP user timeout (see RFC5482) */ 21608d1b8c62STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, 21618d1b8c62STrond Myklebust (char *)&timeo, sizeof(timeo)); 21628d1b8c62STrond Myklebust } 21638d1b8c62STrond Myklebust 21647196dbb0STrond Myklebust static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt, 21657196dbb0STrond Myklebust unsigned long connect_timeout, 21667196dbb0STrond Myklebust unsigned long reconnect_timeout) 21677196dbb0STrond Myklebust { 21687196dbb0STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21697196dbb0STrond Myklebust struct rpc_timeout to; 21707196dbb0STrond Myklebust unsigned long initval; 21717196dbb0STrond Myklebust 21727196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 21737196dbb0STrond Myklebust if (reconnect_timeout < xprt->max_reconnect_timeout) 21747196dbb0STrond Myklebust xprt->max_reconnect_timeout = reconnect_timeout; 21757196dbb0STrond Myklebust if (connect_timeout < xprt->connect_timeout) { 21767196dbb0STrond Myklebust memcpy(&to, xprt->timeout, sizeof(to)); 21777196dbb0STrond Myklebust initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1); 21787196dbb0STrond Myklebust /* Arbitrary lower limit */ 21797196dbb0STrond Myklebust if (initval < XS_TCP_INIT_REEST_TO << 1) 21807196dbb0STrond Myklebust initval = XS_TCP_INIT_REEST_TO << 1; 21817196dbb0STrond Myklebust to.to_initval = initval; 21827196dbb0STrond Myklebust to.to_maxval = initval; 21837196dbb0STrond Myklebust memcpy(&transport->tcp_timeout, &to, 21847196dbb0STrond Myklebust sizeof(transport->tcp_timeout)); 21857196dbb0STrond Myklebust xprt->timeout = &transport->tcp_timeout; 21867196dbb0STrond Myklebust xprt->connect_timeout = connect_timeout; 21877196dbb0STrond Myklebust } 21887196dbb0STrond Myklebust set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 21897196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 21907196dbb0STrond Myklebust } 21917196dbb0STrond Myklebust 21928d1b8c62STrond Myklebust static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 21938d1b8c62STrond Myklebust { 21948d1b8c62STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 21958d1b8c62STrond Myklebust int ret = -ENOTCONN; 21968d1b8c62STrond Myklebust 21978d1b8c62STrond Myklebust if (!transport->inet) { 21988d1b8c62STrond Myklebust struct sock *sk = sock->sk; 21998d1b8c62STrond Myklebust unsigned int addr_pref = IPV6_PREFER_SRC_PUBLIC; 22008d1b8c62STrond Myklebust 2201d88e4d82SNeilBrown /* Avoid temporary address, they are bad for long-lived 2202d88e4d82SNeilBrown * connections such as NFS mounts. 2203d88e4d82SNeilBrown * RFC4941, section 3.6 suggests that: 2204d88e4d82SNeilBrown * Individual applications, which have specific 2205d88e4d82SNeilBrown * knowledge about the normal duration of connections, 2206d88e4d82SNeilBrown * MAY override this as appropriate. 2207d88e4d82SNeilBrown */ 2208d88e4d82SNeilBrown kernel_setsockopt(sock, SOL_IPV6, IPV6_ADDR_PREFERENCES, 2209d88e4d82SNeilBrown (char *)&addr_pref, sizeof(addr_pref)); 2210d88e4d82SNeilBrown 22118d1b8c62STrond Myklebust xs_tcp_set_socket_timeouts(xprt, sock); 2212775f06abSTrond Myklebust 2213b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2214b0d93ad5SChuck Lever 22152a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 22162a9e1cfaSTrond Myklebust 2217b0d93ad5SChuck Lever sk->sk_user_data = xprt; 22185157b956STrond Myklebust sk->sk_data_ready = xs_data_ready; 2219b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change; 2220b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space; 2221b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 22222118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 2223c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 22243167e12cSChuck Lever 22253167e12cSChuck Lever /* socket options */ 22263167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER); 22273167e12cSChuck Lever tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF; 2228b0d93ad5SChuck Lever 2229b0d93ad5SChuck Lever xprt_clear_connected(xprt); 2230b0d93ad5SChuck Lever 2231b0d93ad5SChuck Lever /* Reset to new socket */ 2232ee0ac0c2SChuck Lever transport->sock = sock; 2233ee0ac0c2SChuck Lever transport->inet = sk; 2234b0d93ad5SChuck Lever 2235b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2236b0d93ad5SChuck Lever } 2237b0d93ad5SChuck Lever 223801d37c42STrond Myklebust if (!xprt_bound(xprt)) 2239fe19a96bSTrond Myklebust goto out; 224001d37c42STrond Myklebust 2241a564b8f0SMel Gorman xs_set_memalloc(xprt); 2242a564b8f0SMel Gorman 2243e1806c7bSTrond Myklebust /* Reset TCP record info */ 2244550aebfeSTrond Myklebust xs_stream_reset_connect(transport); 2245e1806c7bSTrond Myklebust 2246b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */ 22470fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 2248fe19a96bSTrond Myklebust ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK); 2249fe19a96bSTrond Myklebust switch (ret) { 2250fe19a96bSTrond Myklebust case 0: 22514dda9c8aSTrond Myklebust xs_set_srcport(transport, sock); 2252e9d47639SGustavo A. R. Silva /* fall through */ 2253fe19a96bSTrond Myklebust case -EINPROGRESS: 2254fe19a96bSTrond Myklebust /* SYN_SENT! */ 2255fe19a96bSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 2256fe19a96bSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 22571f4c17a0STrond Myklebust break; 22581f4c17a0STrond Myklebust case -EADDRNOTAVAIL: 22591f4c17a0STrond Myklebust /* Source port number is unavailable. Try a new one! */ 22601f4c17a0STrond Myklebust transport->srcport = 0; 2261fe19a96bSTrond Myklebust } 2262fe19a96bSTrond Myklebust out: 2263fe19a96bSTrond Myklebust return ret; 226416be2d20SChuck Lever } 226516be2d20SChuck Lever 226616be2d20SChuck Lever /** 2267b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint 226816be2d20SChuck Lever * 226916be2d20SChuck Lever * Invoked by a work queue tasklet. 227016be2d20SChuck Lever */ 2271cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work) 227216be2d20SChuck Lever { 2273cdd518d5SPavel Emelyanov struct sock_xprt *transport = 2274cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work); 227516be2d20SChuck Lever struct socket *sock = transport->sock; 2276a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt; 2277b61d59ffSTrond Myklebust int status = -EIO; 227816be2d20SChuck Lever 227916be2d20SChuck Lever if (!sock) { 2280cdd518d5SPavel Emelyanov sock = xs_create_sock(xprt, transport, 22814dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_STREAM, 22824dda9c8aSTrond Myklebust IPPROTO_TCP, true); 2283b61d59ffSTrond Myklebust if (IS_ERR(sock)) { 2284b61d59ffSTrond Myklebust status = PTR_ERR(sock); 228516be2d20SChuck Lever goto out; 228616be2d20SChuck Lever } 22877d1e8255STrond Myklebust } 22887d1e8255STrond Myklebust 2289c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2290c740eff8SChuck Lever "%s (port %s)\n", xprt, 2291c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2292c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2293c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 229416be2d20SChuck Lever 229516be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock); 229640b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2297a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n", 229846121cf7SChuck Lever xprt, -status, xprt_connected(xprt), 229946121cf7SChuck Lever sock->sk->sk_state); 2300a246b010SChuck Lever switch (status) { 2301f75e6745STrond Myklebust default: 2302f75e6745STrond Myklebust printk("%s: connect returned unhandled error %d\n", 2303f75e6745STrond Myklebust __func__, status); 2304e9d47639SGustavo A. R. Silva /* fall through */ 2305f75e6745STrond Myklebust case -EADDRNOTAVAIL: 2306f75e6745STrond Myklebust /* We're probably in TIME_WAIT. Get rid of existing socket, 2307f75e6745STrond Myklebust * and retry 2308f75e6745STrond Myklebust */ 2309a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 231088b5ed73STrond Myklebust break; 23112a491991STrond Myklebust case 0: 2312a246b010SChuck Lever case -EINPROGRESS: 2313a246b010SChuck Lever case -EALREADY: 2314718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 23157d1e8255STrond Myklebust return; 23169fcfe0c8STrond Myklebust case -EINVAL: 23179fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link 23189fcfe0c8STrond Myklebust * local IPv6 address without a scope-id. 23199fcfe0c8STrond Myklebust */ 23203ed5e2a2STrond Myklebust case -ECONNREFUSED: 23213ed5e2a2STrond Myklebust case -ECONNRESET: 2322eb5b46faSTrond Myklebust case -ENETDOWN: 23233ed5e2a2STrond Myklebust case -ENETUNREACH: 23244ba161a7STrond Myklebust case -EHOSTUNREACH: 23253913c78cSTrond Myklebust case -EADDRINUSE: 23263601c4a9STrond Myklebust case -ENOBUFS: 23276ea44adcSNeilBrown /* 23286ea44adcSNeilBrown * xs_tcp_force_close() wakes tasks with -EIO. 23296ea44adcSNeilBrown * We need to wake them first to ensure the 23306ea44adcSNeilBrown * correct error code. 23316ea44adcSNeilBrown */ 23326ea44adcSNeilBrown xprt_wake_pending_tasks(xprt, status); 23334efdd92cSTrond Myklebust xs_tcp_force_close(xprt); 23349fcfe0c8STrond Myklebust goto out; 23358a2cec29STrond Myklebust } 23362a491991STrond Myklebust status = -EAGAIN; 2337a246b010SChuck Lever out: 2338718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 23392226feb6SChuck Lever xprt_clear_connecting(xprt); 23407d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2341a246b010SChuck Lever } 2342a246b010SChuck Lever 234302910177STrond Myklebust static unsigned long xs_reconnect_delay(const struct rpc_xprt *xprt) 234402910177STrond Myklebust { 234502910177STrond Myklebust unsigned long start, now = jiffies; 234602910177STrond Myklebust 234702910177STrond Myklebust start = xprt->stat.connect_start + xprt->reestablish_timeout; 234802910177STrond Myklebust if (time_after(start, now)) 234902910177STrond Myklebust return start - now; 235002910177STrond Myklebust return 0; 235102910177STrond Myklebust } 235202910177STrond Myklebust 23533851f1cdSTrond Myklebust static void xs_reconnect_backoff(struct rpc_xprt *xprt) 23543851f1cdSTrond Myklebust { 23553851f1cdSTrond Myklebust xprt->reestablish_timeout <<= 1; 23563851f1cdSTrond Myklebust if (xprt->reestablish_timeout > xprt->max_reconnect_timeout) 23573851f1cdSTrond Myklebust xprt->reestablish_timeout = xprt->max_reconnect_timeout; 23583851f1cdSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 23593851f1cdSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 23603851f1cdSTrond Myklebust } 23613851f1cdSTrond Myklebust 236268e220bdSChuck Lever /** 23639903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint 23641b092092STrond Myklebust * @xprt: pointer to transport structure 23659903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request 23669903cd1cSChuck Lever * 23679903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting. 236803bf4b70SChuck Lever * 236903bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway 237003bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a 237103bf4b70SChuck Lever * socket on a privileged port. 237203bf4b70SChuck Lever * 237303bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents 237403bf4b70SChuck Lever * retry floods (hard mounts). 23759903cd1cSChuck Lever */ 23761b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2377a246b010SChuck Lever { 2378ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 237902910177STrond Myklebust unsigned long delay = 0; 2380a246b010SChuck Lever 2381718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport)); 2382718ba5b8STrond Myklebust 238399b1a4c3STrond Myklebust if (transport->sock != NULL) { 238446121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu " 238546121cf7SChuck Lever "seconds\n", 238603bf4b70SChuck Lever xprt, xprt->reestablish_timeout / HZ); 238799b1a4c3STrond Myklebust 238899b1a4c3STrond Myklebust /* Start by resetting any existing state */ 238999b1a4c3STrond Myklebust xs_reset_transport(transport); 239099b1a4c3STrond Myklebust 239102910177STrond Myklebust delay = xs_reconnect_delay(xprt); 23923851f1cdSTrond Myklebust xs_reconnect_backoff(xprt); 239302910177STrond Myklebust 239402910177STrond Myklebust } else 23959903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 239602910177STrond Myklebust 239740a5f1b1STrond Myklebust queue_delayed_work(xprtiod_workqueue, 239802910177STrond Myklebust &transport->connect_worker, 239902910177STrond Myklebust delay); 2400a246b010SChuck Lever } 2401a246b010SChuck Lever 2402262ca07dSChuck Lever /** 2403176e21eeSChuck Lever * xs_local_print_stats - display AF_LOCAL socket-specifc stats 2404176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics 2405176e21eeSChuck Lever * @seq: output file 2406176e21eeSChuck Lever * 2407176e21eeSChuck Lever */ 2408176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2409176e21eeSChuck Lever { 2410176e21eeSChuck Lever long idle_time = 0; 2411176e21eeSChuck Lever 2412176e21eeSChuck Lever if (xprt_connected(xprt)) 2413176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2414176e21eeSChuck Lever 2415176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu " 241615a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2417176e21eeSChuck Lever xprt->stat.bind_count, 2418176e21eeSChuck Lever xprt->stat.connect_count, 24198440a886SChuck Lever xprt->stat.connect_time / HZ, 2420176e21eeSChuck Lever idle_time, 2421176e21eeSChuck Lever xprt->stat.sends, 2422176e21eeSChuck Lever xprt->stat.recvs, 2423176e21eeSChuck Lever xprt->stat.bad_xids, 2424176e21eeSChuck Lever xprt->stat.req_u, 242515a45206SAndy Adamson xprt->stat.bklog_u, 242615a45206SAndy Adamson xprt->stat.max_slots, 242715a45206SAndy Adamson xprt->stat.sending_u, 242815a45206SAndy Adamson xprt->stat.pending_u); 2429176e21eeSChuck Lever } 2430176e21eeSChuck Lever 2431176e21eeSChuck Lever /** 2432262ca07dSChuck Lever * xs_udp_print_stats - display UDP socket-specifc stats 2433262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2434262ca07dSChuck Lever * @seq: output file 2435262ca07dSChuck Lever * 2436262ca07dSChuck Lever */ 2437262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2438262ca07dSChuck Lever { 2439c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2440c8475461SChuck Lever 244115a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu " 244215a45206SAndy Adamson "%lu %llu %llu\n", 2443fbfffbd5SChuck Lever transport->srcport, 2444262ca07dSChuck Lever xprt->stat.bind_count, 2445262ca07dSChuck Lever xprt->stat.sends, 2446262ca07dSChuck Lever xprt->stat.recvs, 2447262ca07dSChuck Lever xprt->stat.bad_xids, 2448262ca07dSChuck Lever xprt->stat.req_u, 244915a45206SAndy Adamson xprt->stat.bklog_u, 245015a45206SAndy Adamson xprt->stat.max_slots, 245115a45206SAndy Adamson xprt->stat.sending_u, 245215a45206SAndy Adamson xprt->stat.pending_u); 2453262ca07dSChuck Lever } 2454262ca07dSChuck Lever 2455262ca07dSChuck Lever /** 2456262ca07dSChuck Lever * xs_tcp_print_stats - display TCP socket-specifc stats 2457262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2458262ca07dSChuck Lever * @seq: output file 2459262ca07dSChuck Lever * 2460262ca07dSChuck Lever */ 2461262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2462262ca07dSChuck Lever { 2463c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2464262ca07dSChuck Lever long idle_time = 0; 2465262ca07dSChuck Lever 2466262ca07dSChuck Lever if (xprt_connected(xprt)) 2467262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2468262ca07dSChuck Lever 246915a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu " 247015a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2471fbfffbd5SChuck Lever transport->srcport, 2472262ca07dSChuck Lever xprt->stat.bind_count, 2473262ca07dSChuck Lever xprt->stat.connect_count, 24748440a886SChuck Lever xprt->stat.connect_time / HZ, 2475262ca07dSChuck Lever idle_time, 2476262ca07dSChuck Lever xprt->stat.sends, 2477262ca07dSChuck Lever xprt->stat.recvs, 2478262ca07dSChuck Lever xprt->stat.bad_xids, 2479262ca07dSChuck Lever xprt->stat.req_u, 248015a45206SAndy Adamson xprt->stat.bklog_u, 248115a45206SAndy Adamson xprt->stat.max_slots, 248215a45206SAndy Adamson xprt->stat.sending_u, 248315a45206SAndy Adamson xprt->stat.pending_u); 2484262ca07dSChuck Lever } 2485262ca07dSChuck Lever 24864cfc7e60SRahul Iyer /* 24874cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason 24884cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want 24894cfc7e60SRahul Iyer * to use the server side send routines. 24904cfc7e60SRahul Iyer */ 24915fe6eaa1SChuck Lever static int bc_malloc(struct rpc_task *task) 24924cfc7e60SRahul Iyer { 24935fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp; 24945fe6eaa1SChuck Lever size_t size = rqst->rq_callsize; 24954cfc7e60SRahul Iyer struct page *page; 24964cfc7e60SRahul Iyer struct rpc_buffer *buf; 24974cfc7e60SRahul Iyer 24985fe6eaa1SChuck Lever if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) { 24995fe6eaa1SChuck Lever WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n", 25005fe6eaa1SChuck Lever size); 25015fe6eaa1SChuck Lever return -EINVAL; 25025fe6eaa1SChuck Lever } 25034cfc7e60SRahul Iyer 2504b8a13d03SWeston Andros Adamson page = alloc_page(GFP_KERNEL); 25054cfc7e60SRahul Iyer if (!page) 25065fe6eaa1SChuck Lever return -ENOMEM; 25074cfc7e60SRahul Iyer 25084cfc7e60SRahul Iyer buf = page_address(page); 25094cfc7e60SRahul Iyer buf->len = PAGE_SIZE; 25104cfc7e60SRahul Iyer 25115fe6eaa1SChuck Lever rqst->rq_buffer = buf->data; 251218e601d6SJeff Layton rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; 25135fe6eaa1SChuck Lever return 0; 25144cfc7e60SRahul Iyer } 25154cfc7e60SRahul Iyer 25164cfc7e60SRahul Iyer /* 25174cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine 25184cfc7e60SRahul Iyer */ 25193435c74aSChuck Lever static void bc_free(struct rpc_task *task) 25204cfc7e60SRahul Iyer { 25213435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer; 25224cfc7e60SRahul Iyer struct rpc_buffer *buf; 25234cfc7e60SRahul Iyer 25244cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data); 25254cfc7e60SRahul Iyer free_page((unsigned long)buf); 25264cfc7e60SRahul Iyer } 25274cfc7e60SRahul Iyer 25284cfc7e60SRahul Iyer /* 25294cfc7e60SRahul Iyer * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex 25304cfc7e60SRahul Iyer * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request. 25314cfc7e60SRahul Iyer */ 25324cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req) 25334cfc7e60SRahul Iyer { 25344cfc7e60SRahul Iyer int len; 25354cfc7e60SRahul Iyer struct xdr_buf *xbufp = &req->rq_snd_buf; 25364cfc7e60SRahul Iyer struct rpc_xprt *xprt = req->rq_xprt; 25374cfc7e60SRahul Iyer struct sock_xprt *transport = 25384cfc7e60SRahul Iyer container_of(xprt, struct sock_xprt, xprt); 25394cfc7e60SRahul Iyer struct socket *sock = transport->sock; 25404cfc7e60SRahul Iyer unsigned long headoff; 25414cfc7e60SRahul Iyer unsigned long tailoff; 25424cfc7e60SRahul Iyer 254361677eeeSChuck Lever xs_encode_stream_record_marker(xbufp); 25444cfc7e60SRahul Iyer 25454cfc7e60SRahul Iyer tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK; 25464cfc7e60SRahul Iyer headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK; 25474cfc7e60SRahul Iyer len = svc_send_common(sock, xbufp, 25484cfc7e60SRahul Iyer virt_to_page(xbufp->head[0].iov_base), headoff, 25494cfc7e60SRahul Iyer xbufp->tail[0].iov_base, tailoff); 25504cfc7e60SRahul Iyer 25514cfc7e60SRahul Iyer if (len != xbufp->len) { 25524cfc7e60SRahul Iyer printk(KERN_NOTICE "Error sending entire callback!\n"); 25534cfc7e60SRahul Iyer len = -EAGAIN; 25544cfc7e60SRahul Iyer } 25554cfc7e60SRahul Iyer 25564cfc7e60SRahul Iyer return len; 25574cfc7e60SRahul Iyer } 25584cfc7e60SRahul Iyer 25594cfc7e60SRahul Iyer /* 25604cfc7e60SRahul Iyer * The send routine. Borrows from svc_send 25614cfc7e60SRahul Iyer */ 2562adfa7144STrond Myklebust static int bc_send_request(struct rpc_rqst *req) 25634cfc7e60SRahul Iyer { 25644cfc7e60SRahul Iyer struct svc_xprt *xprt; 25657fc56136SAndrzej Hajda int len; 25664cfc7e60SRahul Iyer 25674cfc7e60SRahul Iyer dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid)); 25684cfc7e60SRahul Iyer /* 25694cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt 25704cfc7e60SRahul Iyer */ 25714cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt; 25724cfc7e60SRahul Iyer 25734cfc7e60SRahul Iyer /* 25744cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared 25754cfc7e60SRahul Iyer * with the fore channel 25764cfc7e60SRahul Iyer */ 2577c544577dSTrond Myklebust mutex_lock(&xprt->xpt_mutex); 25784cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags)) 25794cfc7e60SRahul Iyer len = -ENOTCONN; 25804cfc7e60SRahul Iyer else 25814cfc7e60SRahul Iyer len = bc_sendto(req); 25824cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex); 25834cfc7e60SRahul Iyer 25844cfc7e60SRahul Iyer if (len > 0) 25854cfc7e60SRahul Iyer len = 0; 25864cfc7e60SRahul Iyer 25874cfc7e60SRahul Iyer return len; 25884cfc7e60SRahul Iyer } 25894cfc7e60SRahul Iyer 25904cfc7e60SRahul Iyer /* 25914cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing 25924cfc7e60SRahul Iyer */ 25934cfc7e60SRahul Iyer 25944cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt) 25954cfc7e60SRahul Iyer { 25964cfc7e60SRahul Iyer } 25974cfc7e60SRahul Iyer 25984cfc7e60SRahul Iyer /* 25994cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client 26004cfc7e60SRahul Iyer * initiated, we do nothing 26014cfc7e60SRahul Iyer */ 26024cfc7e60SRahul Iyer 26034cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt) 26044cfc7e60SRahul Iyer { 260547f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt); 260647f72efaSKinglong Mee 260747f72efaSKinglong Mee xs_xprt_free(xprt); 260847f72efaSKinglong Mee module_put(THIS_MODULE); 26094cfc7e60SRahul Iyer } 26104cfc7e60SRahul Iyer 2611d31ae254SChuck Lever static const struct rpc_xprt_ops xs_local_ops = { 2612176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt, 26134cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 2614f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2615a9cde23aSChuck Lever .free_slot = xprt_free_slot, 2616176e21eeSChuck Lever .rpcbind = xs_local_rpcbind, 2617176e21eeSChuck Lever .set_port = xs_local_set_port, 2618dc107402SJ. Bruce Fields .connect = xs_local_connect, 2619176e21eeSChuck Lever .buf_alloc = rpc_malloc, 2620176e21eeSChuck Lever .buf_free = rpc_free, 2621550aebfeSTrond Myklebust .prepare_request = xs_stream_prepare_request, 2622176e21eeSChuck Lever .send_request = xs_local_send_request, 2623176e21eeSChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2624176e21eeSChuck Lever .close = xs_close, 2625a1311d87STrond Myklebust .destroy = xs_destroy, 2626176e21eeSChuck Lever .print_stats = xs_local_print_stats, 2627d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2628d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 2629176e21eeSChuck Lever }; 2630176e21eeSChuck Lever 2631d31ae254SChuck Lever static const struct rpc_xprt_ops xs_udp_ops = { 263243118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size, 263312a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong, 263449e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong, 2635f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2636a9cde23aSChuck Lever .free_slot = xprt_free_slot, 263745160d62SChuck Lever .rpcbind = rpcb_getport_async, 263892200412SChuck Lever .set_port = xs_set_port, 26399903cd1cSChuck Lever .connect = xs_connect, 264002107148SChuck Lever .buf_alloc = rpc_malloc, 264102107148SChuck Lever .buf_free = rpc_free, 2642262965f5SChuck Lever .send_request = xs_udp_send_request, 2643fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_rtt, 264446c0ee8bSChuck Lever .timer = xs_udp_timer, 2645a58dd398SChuck Lever .release_request = xprt_release_rqst_cong, 2646262965f5SChuck Lever .close = xs_close, 2647262965f5SChuck Lever .destroy = xs_destroy, 2648262ca07dSChuck Lever .print_stats = xs_udp_print_stats, 2649d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2650d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26514a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2652262965f5SChuck Lever }; 2653262965f5SChuck Lever 2654d31ae254SChuck Lever static const struct rpc_xprt_ops xs_tcp_ops = { 265512a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt, 26564cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 265736bd7de9STrond Myklebust .alloc_slot = xprt_alloc_slot, 2658a9cde23aSChuck Lever .free_slot = xprt_free_slot, 265945160d62SChuck Lever .rpcbind = rpcb_getport_async, 266092200412SChuck Lever .set_port = xs_set_port, 26610b9e7943STrond Myklebust .connect = xs_connect, 266202107148SChuck Lever .buf_alloc = rpc_malloc, 266302107148SChuck Lever .buf_free = rpc_free, 2664277e4ab7STrond Myklebust .prepare_request = xs_stream_prepare_request, 2665262965f5SChuck Lever .send_request = xs_tcp_send_request, 2666fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2667c627d31bSTrond Myklebust .close = xs_tcp_shutdown, 26689903cd1cSChuck Lever .destroy = xs_destroy, 26697196dbb0STrond Myklebust .set_connect_timeout = xs_tcp_set_connect_timeout, 2670262ca07dSChuck Lever .print_stats = xs_tcp_print_stats, 2671d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2672d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 26734a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 267442e5c3e2SChuck Lever #ifdef CONFIG_SUNRPC_BACKCHANNEL 267542e5c3e2SChuck Lever .bc_setup = xprt_setup_bc, 267676566773SChuck Lever .bc_up = xs_tcp_bc_up, 26776b26cc8cSChuck Lever .bc_maxpayload = xs_tcp_bc_maxpayload, 267842e5c3e2SChuck Lever .bc_free_rqst = xprt_free_bc_rqst, 267942e5c3e2SChuck Lever .bc_destroy = xprt_destroy_bc, 268042e5c3e2SChuck Lever #endif 2681a246b010SChuck Lever }; 2682a246b010SChuck Lever 26834cfc7e60SRahul Iyer /* 26844cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel 26854cfc7e60SRahul Iyer */ 26864cfc7e60SRahul Iyer 2687d31ae254SChuck Lever static const struct rpc_xprt_ops bc_tcp_ops = { 26884cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt, 26894cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt, 269084e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot, 2691a9cde23aSChuck Lever .free_slot = xprt_free_slot, 26924cfc7e60SRahul Iyer .buf_alloc = bc_malloc, 26934cfc7e60SRahul Iyer .buf_free = bc_free, 26944cfc7e60SRahul Iyer .send_request = bc_send_request, 26954cfc7e60SRahul Iyer .set_retrans_timeout = xprt_set_retrans_timeout_def, 26964cfc7e60SRahul Iyer .close = bc_close, 26974cfc7e60SRahul Iyer .destroy = bc_destroy, 26984cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats, 2699d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2700d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 27014a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 27024cfc7e60SRahul Iyer }; 27034cfc7e60SRahul Iyer 270492476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap) 270592476850SChuck Lever { 270692476850SChuck Lever static const struct sockaddr_in sin = { 270792476850SChuck Lever .sin_family = AF_INET, 270892476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY), 270992476850SChuck Lever }; 271092476850SChuck Lever static const struct sockaddr_in6 sin6 = { 271192476850SChuck Lever .sin6_family = AF_INET6, 271292476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT, 271392476850SChuck Lever }; 271492476850SChuck Lever 271592476850SChuck Lever switch (family) { 2716176e21eeSChuck Lever case AF_LOCAL: 2717176e21eeSChuck Lever break; 271892476850SChuck Lever case AF_INET: 271992476850SChuck Lever memcpy(sap, &sin, sizeof(sin)); 272092476850SChuck Lever break; 272192476850SChuck Lever case AF_INET6: 272292476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6)); 272392476850SChuck Lever break; 272492476850SChuck Lever default: 272592476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__); 272692476850SChuck Lever return -EAFNOSUPPORT; 272792476850SChuck Lever } 272892476850SChuck Lever return 0; 272992476850SChuck Lever } 273092476850SChuck Lever 27313c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args, 2732d9ba131dSTrond Myklebust unsigned int slot_table_size, 2733d9ba131dSTrond Myklebust unsigned int max_slot_table_size) 2734c8541ecdSChuck Lever { 2735c8541ecdSChuck Lever struct rpc_xprt *xprt; 2736ffc2e518SChuck Lever struct sock_xprt *new; 2737c8541ecdSChuck Lever 273896802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) { 2739c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n"); 2740c8541ecdSChuck Lever return ERR_PTR(-EBADF); 2741c8541ecdSChuck Lever } 2742c8541ecdSChuck Lever 2743d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size, 2744d9ba131dSTrond Myklebust max_slot_table_size); 2745bd1722d4SPavel Emelyanov if (xprt == NULL) { 274646121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate " 274746121cf7SChuck Lever "rpc_xprt\n"); 2748c8541ecdSChuck Lever return ERR_PTR(-ENOMEM); 2749c8541ecdSChuck Lever } 2750c8541ecdSChuck Lever 2751bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt); 2752edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex); 275396802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen); 275496802a09SFrank van Maarseveen xprt->addrlen = args->addrlen; 2755d3bc9a1dSFrank van Maarseveen if (args->srcaddr) 2756fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen); 275792476850SChuck Lever else { 275892476850SChuck Lever int err; 275992476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family, 276092476850SChuck Lever (struct sockaddr *)&new->srcaddr); 27612aa13531SStanislav Kinsbursky if (err != 0) { 27622aa13531SStanislav Kinsbursky xprt_free(xprt); 276392476850SChuck Lever return ERR_PTR(err); 276492476850SChuck Lever } 27652aa13531SStanislav Kinsbursky } 2766c8541ecdSChuck Lever 2767c8541ecdSChuck Lever return xprt; 2768c8541ecdSChuck Lever } 2769c8541ecdSChuck Lever 2770176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = { 2771176e21eeSChuck Lever .to_initval = 10 * HZ, 2772176e21eeSChuck Lever .to_maxval = 10 * HZ, 2773176e21eeSChuck Lever .to_retries = 2, 2774176e21eeSChuck Lever }; 2775176e21eeSChuck Lever 2776176e21eeSChuck Lever /** 2777176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket 2778176e21eeSChuck Lever * @args: rpc transport creation arguments 2779176e21eeSChuck Lever * 2780176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP 2781176e21eeSChuck Lever */ 2782176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args) 2783176e21eeSChuck Lever { 2784176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr; 2785176e21eeSChuck Lever struct sock_xprt *transport; 2786176e21eeSChuck Lever struct rpc_xprt *xprt; 2787176e21eeSChuck Lever struct rpc_xprt *ret; 2788176e21eeSChuck Lever 2789d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2790d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries); 2791176e21eeSChuck Lever if (IS_ERR(xprt)) 2792176e21eeSChuck Lever return xprt; 2793176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2794176e21eeSChuck Lever 2795176e21eeSChuck Lever xprt->prot = 0; 2796176e21eeSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2797176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2798176e21eeSChuck Lever 2799176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO; 2800176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2801176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2802176e21eeSChuck Lever 2803176e21eeSChuck Lever xprt->ops = &xs_local_ops; 2804176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout; 2805176e21eeSChuck Lever 2806550aebfeSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 2807550aebfeSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_dummy_setup_socket); 280893dc41bdSNeilBrown 2809176e21eeSChuck Lever switch (sun->sun_family) { 2810176e21eeSChuck Lever case AF_LOCAL: 2811176e21eeSChuck Lever if (sun->sun_path[0] != '/') { 2812176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n", 2813176e21eeSChuck Lever sun->sun_path); 2814176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2815176e21eeSChuck Lever goto out_err; 2816176e21eeSChuck Lever } 2817176e21eeSChuck Lever xprt_set_bound(xprt); 2818176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL); 28197073ea87SJ. Bruce Fields ret = ERR_PTR(xs_local_setup_socket(transport)); 28207073ea87SJ. Bruce Fields if (ret) 28217073ea87SJ. Bruce Fields goto out_err; 2822176e21eeSChuck Lever break; 2823176e21eeSChuck Lever default: 2824176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT); 2825176e21eeSChuck Lever goto out_err; 2826176e21eeSChuck Lever } 2827176e21eeSChuck Lever 2828176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n", 2829176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2830176e21eeSChuck Lever 2831176e21eeSChuck Lever if (try_module_get(THIS_MODULE)) 2832176e21eeSChuck Lever return xprt; 2833176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2834176e21eeSChuck Lever out_err: 2835315f3812SKinglong Mee xs_xprt_free(xprt); 2836176e21eeSChuck Lever return ret; 2837176e21eeSChuck Lever } 2838176e21eeSChuck Lever 28392881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = { 28402881ae74STrond Myklebust .to_initval = 5 * HZ, 28412881ae74STrond Myklebust .to_maxval = 30 * HZ, 28422881ae74STrond Myklebust .to_increment = 5 * HZ, 28432881ae74STrond Myklebust .to_retries = 5, 28442881ae74STrond Myklebust }; 28452881ae74STrond Myklebust 28469903cd1cSChuck Lever /** 28479903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket 284896802a09SFrank van Maarseveen * @args: rpc transport creation arguments 28499903cd1cSChuck Lever * 28509903cd1cSChuck Lever */ 2851483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args) 2852a246b010SChuck Lever { 28538f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2854c8541ecdSChuck Lever struct rpc_xprt *xprt; 2855c8475461SChuck Lever struct sock_xprt *transport; 28560a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2857a246b010SChuck Lever 2858d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries, 2859d9ba131dSTrond Myklebust xprt_udp_slot_table_entries); 2860c8541ecdSChuck Lever if (IS_ERR(xprt)) 2861c8541ecdSChuck Lever return xprt; 2862c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2863a246b010SChuck Lever 2864ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP; 2865808012fbSChuck Lever xprt->tsh_size = 0; 2866a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */ 2867a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3); 2868a246b010SChuck Lever 286903bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 287003bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO; 287103bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2872a246b010SChuck Lever 2873262965f5SChuck Lever xprt->ops = &xs_udp_ops; 2874a246b010SChuck Lever 2875ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout; 2876a246b010SChuck Lever 2877f9b2ee71STrond Myklebust INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn); 2878edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket); 2879edc1b01cSTrond Myklebust 28808f9d5b1aSChuck Lever switch (addr->sa_family) { 28818f9d5b1aSChuck Lever case AF_INET: 28828f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 28838f9d5b1aSChuck Lever xprt_set_bound(xprt); 28848f9d5b1aSChuck Lever 28859dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP); 28868f9d5b1aSChuck Lever break; 28878f9d5b1aSChuck Lever case AF_INET6: 28888f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 28898f9d5b1aSChuck Lever xprt_set_bound(xprt); 28908f9d5b1aSChuck Lever 28919dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6); 28928f9d5b1aSChuck Lever break; 28938f9d5b1aSChuck Lever default: 28940a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 28950a68b0beSJ. Bruce Fields goto out_err; 28968f9d5b1aSChuck Lever } 28978f9d5b1aSChuck Lever 2898c740eff8SChuck Lever if (xprt_bound(xprt)) 2899c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2900c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2901c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2902c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2903c740eff8SChuck Lever else 2904c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2905c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2906c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2907edb267a6SChuck Lever 2908bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2909c8541ecdSChuck Lever return xprt; 29100a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29110a68b0beSJ. Bruce Fields out_err: 2912315f3812SKinglong Mee xs_xprt_free(xprt); 29130a68b0beSJ. Bruce Fields return ret; 2914a246b010SChuck Lever } 2915a246b010SChuck Lever 29162881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = { 29172881ae74STrond Myklebust .to_initval = 60 * HZ, 29182881ae74STrond Myklebust .to_maxval = 60 * HZ, 29192881ae74STrond Myklebust .to_retries = 2, 29202881ae74STrond Myklebust }; 29212881ae74STrond Myklebust 29229903cd1cSChuck Lever /** 29239903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket 292496802a09SFrank van Maarseveen * @args: rpc transport creation arguments 29259903cd1cSChuck Lever * 29269903cd1cSChuck Lever */ 2927483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args) 2928a246b010SChuck Lever { 29298f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 2930c8541ecdSChuck Lever struct rpc_xprt *xprt; 2931c8475461SChuck Lever struct sock_xprt *transport; 29320a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 2933b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries; 2934b7993cebSTrond Myklebust 2935b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS) 2936b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT; 2937a246b010SChuck Lever 2938d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2939b7993cebSTrond Myklebust max_slot_table_size); 2940c8541ecdSChuck Lever if (IS_ERR(xprt)) 2941c8541ecdSChuck Lever return xprt; 2942c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2943a246b010SChuck Lever 2944ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP; 2945808012fbSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2946808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2947a246b010SChuck Lever 294803bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 294903bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 295003bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2951a246b010SChuck Lever 2952262965f5SChuck Lever xprt->ops = &xs_tcp_ops; 2953ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout; 2954a246b010SChuck Lever 29553851f1cdSTrond Myklebust xprt->max_reconnect_timeout = xprt->timeout->to_maxval; 29567196dbb0STrond Myklebust xprt->connect_timeout = xprt->timeout->to_initval * 29577196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 29583851f1cdSTrond Myklebust 2959c50b8ee0STrond Myklebust INIT_WORK(&transport->recv_worker, xs_stream_data_receive_workfn); 2960edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket); 2961edc1b01cSTrond Myklebust 29628f9d5b1aSChuck Lever switch (addr->sa_family) { 29638f9d5b1aSChuck Lever case AF_INET: 29648f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 29658f9d5b1aSChuck Lever xprt_set_bound(xprt); 29668f9d5b1aSChuck Lever 29679dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP); 29688f9d5b1aSChuck Lever break; 29698f9d5b1aSChuck Lever case AF_INET6: 29708f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 29718f9d5b1aSChuck Lever xprt_set_bound(xprt); 29728f9d5b1aSChuck Lever 29739dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6); 29748f9d5b1aSChuck Lever break; 29758f9d5b1aSChuck Lever default: 29760a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 29770a68b0beSJ. Bruce Fields goto out_err; 29788f9d5b1aSChuck Lever } 29798f9d5b1aSChuck Lever 2980c740eff8SChuck Lever if (xprt_bound(xprt)) 2981c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 2982c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2983c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 2984c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2985c740eff8SChuck Lever else 2986c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 2987c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2988c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 2989c740eff8SChuck Lever 2990bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 2991c8541ecdSChuck Lever return xprt; 29920a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 29930a68b0beSJ. Bruce Fields out_err: 2994315f3812SKinglong Mee xs_xprt_free(xprt); 29950a68b0beSJ. Bruce Fields return ret; 2996a246b010SChuck Lever } 2997282b32e1SChuck Lever 2998f300babaSAlexandros Batsakis /** 2999f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket 3000f300babaSAlexandros Batsakis * @args: rpc transport creation arguments 3001f300babaSAlexandros Batsakis * 3002f300babaSAlexandros Batsakis */ 3003f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args) 3004f300babaSAlexandros Batsakis { 3005f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr; 3006f300babaSAlexandros Batsakis struct rpc_xprt *xprt; 3007f300babaSAlexandros Batsakis struct sock_xprt *transport; 3008f300babaSAlexandros Batsakis struct svc_sock *bc_sock; 30090a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 3010f300babaSAlexandros Batsakis 3011d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 3012d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries); 3013f300babaSAlexandros Batsakis if (IS_ERR(xprt)) 3014f300babaSAlexandros Batsakis return xprt; 3015f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt); 3016f300babaSAlexandros Batsakis 3017f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP; 3018f300babaSAlexandros Batsakis xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 3019f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 3020f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout; 3021f300babaSAlexandros Batsakis 3022f300babaSAlexandros Batsakis /* backchannel */ 3023f300babaSAlexandros Batsakis xprt_set_bound(xprt); 3024f300babaSAlexandros Batsakis xprt->bind_timeout = 0; 3025f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0; 3026f300babaSAlexandros Batsakis xprt->idle_timeout = 0; 3027f300babaSAlexandros Batsakis 3028f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops; 3029f300babaSAlexandros Batsakis 3030f300babaSAlexandros Batsakis switch (addr->sa_family) { 3031f300babaSAlexandros Batsakis case AF_INET: 3032f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3033f300babaSAlexandros Batsakis RPCBIND_NETID_TCP); 3034f300babaSAlexandros Batsakis break; 3035f300babaSAlexandros Batsakis case AF_INET6: 3036f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3037f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6); 3038f300babaSAlexandros Batsakis break; 3039f300babaSAlexandros Batsakis default: 30400a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 30410a68b0beSJ. Bruce Fields goto out_err; 3042f300babaSAlexandros Batsakis } 3043f300babaSAlexandros Batsakis 3044f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3045f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR], 3046f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT], 3047f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]); 3048f300babaSAlexandros Batsakis 3049f300babaSAlexandros Batsakis /* 305099de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection, 305128303ca3SWeng Meiling * we want to keep it around as long as the connection lasts, 305228303ca3SWeng Meiling * in case we need to start using it for a backchannel again; 305328303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed. 305499de8ea9SJ. Bruce Fields */ 305599de8ea9SJ. Bruce Fields xprt_get(xprt); 305699de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt; 305799de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt; 305899de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt); 305999de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock; 306099de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk; 306199de8ea9SJ. Bruce Fields 306299de8ea9SJ. Bruce Fields /* 3063f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set 3064f300babaSAlexandros Batsakis * the xprt status to connected 3065f300babaSAlexandros Batsakis */ 3066f300babaSAlexandros Batsakis xprt_set_connected(xprt); 3067f300babaSAlexandros Batsakis 3068f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE)) 3069f300babaSAlexandros Batsakis return xprt; 3070642aab58SKinglong Mee 3071642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL; 307239a9beabSJ. Bruce Fields args->bc_xprt->xpt_bc_xps = NULL; 307399de8ea9SJ. Bruce Fields xprt_put(xprt); 30740a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 30750a68b0beSJ. Bruce Fields out_err: 3076315f3812SKinglong Mee xs_xprt_free(xprt); 30770a68b0beSJ. Bruce Fields return ret; 3078f300babaSAlexandros Batsakis } 3079f300babaSAlexandros Batsakis 3080176e21eeSChuck Lever static struct xprt_class xs_local_transport = { 3081176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list), 3082176e21eeSChuck Lever .name = "named UNIX socket", 3083176e21eeSChuck Lever .owner = THIS_MODULE, 3084176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL, 3085176e21eeSChuck Lever .setup = xs_setup_local, 3086176e21eeSChuck Lever }; 3087176e21eeSChuck Lever 3088bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = { 3089bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list), 3090bc25571eS\"Talpey, Thomas\ .name = "udp", 3091bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3092f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP, 3093bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp, 3094bc25571eS\"Talpey, Thomas\ }; 3095bc25571eS\"Talpey, Thomas\ 3096bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = { 3097bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list), 3098bc25571eS\"Talpey, Thomas\ .name = "tcp", 3099bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3100f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP, 3101bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp, 3102bc25571eS\"Talpey, Thomas\ }; 3103bc25571eS\"Talpey, Thomas\ 3104f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = { 3105f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list), 3106f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel", 3107f300babaSAlexandros Batsakis .owner = THIS_MODULE, 3108f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP, 3109f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp, 3110f300babaSAlexandros Batsakis }; 3111f300babaSAlexandros Batsakis 3112282b32e1SChuck Lever /** 3113bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client 3114282b32e1SChuck Lever * 3115282b32e1SChuck Lever */ 3116282b32e1SChuck Lever int init_socket_xprt(void) 3117282b32e1SChuck Lever { 3118f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 31192b1bec5fSEric W. Biederman if (!sunrpc_table_header) 31200b4d4147SEric W. Biederman sunrpc_table_header = register_sysctl_table(sunrpc_table); 3121fbf76683SChuck Lever #endif 3122fbf76683SChuck Lever 3123176e21eeSChuck Lever xprt_register_transport(&xs_local_transport); 3124bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport); 3125bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport); 3126f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport); 3127bc25571eS\"Talpey, Thomas\ 3128282b32e1SChuck Lever return 0; 3129282b32e1SChuck Lever } 3130282b32e1SChuck Lever 3131282b32e1SChuck Lever /** 3132bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister 3133282b32e1SChuck Lever * 3134282b32e1SChuck Lever */ 3135282b32e1SChuck Lever void cleanup_socket_xprt(void) 3136282b32e1SChuck Lever { 3137f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 3138fbf76683SChuck Lever if (sunrpc_table_header) { 3139fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header); 3140fbf76683SChuck Lever sunrpc_table_header = NULL; 3141fbf76683SChuck Lever } 3142fbf76683SChuck Lever #endif 3143bc25571eS\"Talpey, Thomas\ 3144176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport); 3145bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport); 3146bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport); 3147f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport); 3148282b32e1SChuck Lever } 3149cbf11071STrond Myklebust 31509bbb9e5aSRusty Russell static int param_set_uint_minmax(const char *val, 31519bbb9e5aSRusty Russell const struct kernel_param *kp, 3152cbf11071STrond Myklebust unsigned int min, unsigned int max) 3153cbf11071STrond Myklebust { 315400cfaa94SDaniel Walter unsigned int num; 3155cbf11071STrond Myklebust int ret; 3156cbf11071STrond Myklebust 3157cbf11071STrond Myklebust if (!val) 3158cbf11071STrond Myklebust return -EINVAL; 315900cfaa94SDaniel Walter ret = kstrtouint(val, 0, &num); 31609761a246SDan Carpenter if (ret) 31619761a246SDan Carpenter return ret; 31629761a246SDan Carpenter if (num < min || num > max) 3163cbf11071STrond Myklebust return -EINVAL; 3164cbf11071STrond Myklebust *((unsigned int *)kp->arg) = num; 3165cbf11071STrond Myklebust return 0; 3166cbf11071STrond Myklebust } 3167cbf11071STrond Myklebust 31689bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp) 3169cbf11071STrond Myklebust { 3170cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3171cbf11071STrond Myklebust RPC_MIN_RESVPORT, 3172cbf11071STrond Myklebust RPC_MAX_RESVPORT); 3173cbf11071STrond Myklebust } 3174cbf11071STrond Myklebust 31759c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = { 31769bbb9e5aSRusty Russell .set = param_set_portnr, 31779bbb9e5aSRusty Russell .get = param_get_uint, 31789bbb9e5aSRusty Russell }; 31799bbb9e5aSRusty Russell 3180cbf11071STrond Myklebust #define param_check_portnr(name, p) \ 3181cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3182cbf11071STrond Myklebust 3183cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644); 3184cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644); 3185cbf11071STrond Myklebust 31869bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val, 31879bbb9e5aSRusty Russell const struct kernel_param *kp) 3188cbf11071STrond Myklebust { 3189cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3190cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE, 3191cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE); 3192cbf11071STrond Myklebust } 3193cbf11071STrond Myklebust 31949c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = { 31959bbb9e5aSRusty Russell .set = param_set_slot_table_size, 31969bbb9e5aSRusty Russell .get = param_get_uint, 31979bbb9e5aSRusty Russell }; 31989bbb9e5aSRusty Russell 3199cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \ 3200cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3201cbf11071STrond Myklebust 3202d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val, 3203d9ba131dSTrond Myklebust const struct kernel_param *kp) 3204d9ba131dSTrond Myklebust { 3205d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp, 3206d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE, 3207d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT); 3208d9ba131dSTrond Myklebust } 3209d9ba131dSTrond Myklebust 32109c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = { 3211d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size, 3212d9ba131dSTrond Myklebust .get = param_get_uint, 3213d9ba131dSTrond Myklebust }; 3214d9ba131dSTrond Myklebust 3215d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \ 3216d9ba131dSTrond Myklebust __param_check(name, p, unsigned int); 3217d9ba131dSTrond Myklebust 3218cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries, 3219cbf11071STrond Myklebust slot_table_size, 0644); 3220d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries, 3221d9ba131dSTrond Myklebust max_slot_table_size, 0644); 3222cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries, 3223cbf11071STrond Myklebust slot_table_size, 0644); 3224