1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2a246b010SChuck Lever /* 3a246b010SChuck Lever * linux/net/sunrpc/xprtsock.c 4a246b010SChuck Lever * 5a246b010SChuck Lever * Client-side transport implementation for sockets. 6a246b010SChuck Lever * 7113aa838SAlan Cox * TCP callback races fixes (C) 1998 Red Hat 8113aa838SAlan Cox * TCP send fixes (C) 1998 Red Hat 9a246b010SChuck Lever * TCP NFS related read + write fixes 10a246b010SChuck Lever * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 11a246b010SChuck Lever * 12a246b010SChuck Lever * Rewrite of larges part of the code in order to stabilize TCP stuff. 13a246b010SChuck Lever * Fix behaviour when socket buffer is full. 14a246b010SChuck Lever * (C) 1999 Trond Myklebust <trond.myklebust@fys.uio.no> 1555aa4f58SChuck Lever * 1655aa4f58SChuck Lever * IP socket transport implementation, (C) 2005 Chuck Lever <cel@netapp.com> 178f9d5b1aSChuck Lever * 188f9d5b1aSChuck Lever * IPv6 support contributed by Gilles Quillard, Bull Open Source, 2005. 198f9d5b1aSChuck Lever * <gilles.quillard@bull.net> 20a246b010SChuck Lever */ 21a246b010SChuck Lever 22a246b010SChuck Lever #include <linux/types.h> 23176e21eeSChuck Lever #include <linux/string.h> 24a246b010SChuck Lever #include <linux/slab.h> 25bc25571eS\"Talpey, Thomas\ #include <linux/module.h> 26a246b010SChuck Lever #include <linux/capability.h> 27a246b010SChuck Lever #include <linux/pagemap.h> 28a246b010SChuck Lever #include <linux/errno.h> 29a246b010SChuck Lever #include <linux/socket.h> 30a246b010SChuck Lever #include <linux/in.h> 31a246b010SChuck Lever #include <linux/net.h> 32a246b010SChuck Lever #include <linux/mm.h> 33176e21eeSChuck Lever #include <linux/un.h> 34a246b010SChuck Lever #include <linux/udp.h> 35a246b010SChuck Lever #include <linux/tcp.h> 36a246b010SChuck Lever #include <linux/sunrpc/clnt.h> 375976687aSJeff Layton #include <linux/sunrpc/addr.h> 3802107148SChuck Lever #include <linux/sunrpc/sched.h> 394cfc7e60SRahul Iyer #include <linux/sunrpc/svcsock.h> 4049c36fccS\"Talpey, Thomas\ #include <linux/sunrpc/xprtsock.h> 41a246b010SChuck Lever #include <linux/file.h> 429e00abc3STrond Myklebust #ifdef CONFIG_SUNRPC_BACKCHANNEL 4344b98efdSRicardo Labiaga #include <linux/sunrpc/bc_xprt.h> 4444b98efdSRicardo Labiaga #endif 45a246b010SChuck Lever 46a246b010SChuck Lever #include <net/sock.h> 47a246b010SChuck Lever #include <net/checksum.h> 48a246b010SChuck Lever #include <net/udp.h> 49a246b010SChuck Lever #include <net/tcp.h> 50a246b010SChuck Lever 5140b5ea0cSTrond Myklebust #include <trace/events/sunrpc.h> 5240b5ea0cSTrond Myklebust 534cfc7e60SRahul Iyer #include "sunrpc.h" 54176e21eeSChuck Lever 553d188805STrond Myklebust #define RPC_TCP_READ_CHUNK_SZ (3*512*1024) 563d188805STrond Myklebust 57176e21eeSChuck Lever static void xs_close(struct rpc_xprt *xprt); 587196dbb0STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 597196dbb0STrond Myklebust struct socket *sock); 60176e21eeSChuck Lever 619903cd1cSChuck Lever /* 62c556b754SChuck Lever * xprtsock tunables 63c556b754SChuck Lever */ 6409acfea5STrond Myklebust static unsigned int xprt_udp_slot_table_entries = RPC_DEF_SLOT_TABLE; 6509acfea5STrond Myklebust static unsigned int xprt_tcp_slot_table_entries = RPC_MIN_SLOT_TABLE; 6609acfea5STrond Myklebust static unsigned int xprt_max_tcp_slot_table_entries = RPC_MAX_SLOT_TABLE; 67c556b754SChuck Lever 6809acfea5STrond Myklebust static unsigned int xprt_min_resvport = RPC_DEF_MIN_RESVPORT; 6909acfea5STrond Myklebust static unsigned int xprt_max_resvport = RPC_DEF_MAX_RESVPORT; 70c556b754SChuck Lever 7154c09874STrond Myklebust #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 7254c09874STrond Myklebust 737d1e8255STrond Myklebust #define XS_TCP_LINGER_TO (15U * HZ) 7425fe6142STrond Myklebust static unsigned int xs_tcp_fin_timeout __read_mostly = XS_TCP_LINGER_TO; 757d1e8255STrond Myklebust 76c556b754SChuck Lever /* 77fbf76683SChuck Lever * We can register our own files under /proc/sys/sunrpc by 78fbf76683SChuck Lever * calling register_sysctl_table() again. The files in that 79fbf76683SChuck Lever * directory become the union of all files registered there. 80fbf76683SChuck Lever * 81fbf76683SChuck Lever * We simply need to make sure that we don't collide with 82fbf76683SChuck Lever * someone else's file names! 83fbf76683SChuck Lever */ 84fbf76683SChuck Lever 85fbf76683SChuck Lever static unsigned int min_slot_table_size = RPC_MIN_SLOT_TABLE; 86fbf76683SChuck Lever static unsigned int max_slot_table_size = RPC_MAX_SLOT_TABLE; 87d9ba131dSTrond Myklebust static unsigned int max_tcp_slot_table_limit = RPC_MAX_SLOT_TABLE_LIMIT; 88fbf76683SChuck Lever static unsigned int xprt_min_resvport_limit = RPC_MIN_RESVPORT; 89fbf76683SChuck Lever static unsigned int xprt_max_resvport_limit = RPC_MAX_RESVPORT; 90fbf76683SChuck Lever 91fbf76683SChuck Lever static struct ctl_table_header *sunrpc_table_header; 92fbf76683SChuck Lever 93fbf76683SChuck Lever /* 94fbf76683SChuck Lever * FIXME: changing the UDP slot table size should also resize the UDP 95fbf76683SChuck Lever * socket buffers for existing UDP transports 96fbf76683SChuck Lever */ 97fe2c6338SJoe Perches static struct ctl_table xs_tunables_table[] = { 98fbf76683SChuck Lever { 99fbf76683SChuck Lever .procname = "udp_slot_table_entries", 100fbf76683SChuck Lever .data = &xprt_udp_slot_table_entries, 101fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 102fbf76683SChuck Lever .mode = 0644, 1036d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 104fbf76683SChuck Lever .extra1 = &min_slot_table_size, 105fbf76683SChuck Lever .extra2 = &max_slot_table_size 106fbf76683SChuck Lever }, 107fbf76683SChuck Lever { 108fbf76683SChuck Lever .procname = "tcp_slot_table_entries", 109fbf76683SChuck Lever .data = &xprt_tcp_slot_table_entries, 110fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 111fbf76683SChuck Lever .mode = 0644, 1126d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 113fbf76683SChuck Lever .extra1 = &min_slot_table_size, 114fbf76683SChuck Lever .extra2 = &max_slot_table_size 115fbf76683SChuck Lever }, 116fbf76683SChuck Lever { 117d9ba131dSTrond Myklebust .procname = "tcp_max_slot_table_entries", 118d9ba131dSTrond Myklebust .data = &xprt_max_tcp_slot_table_entries, 119d9ba131dSTrond Myklebust .maxlen = sizeof(unsigned int), 120d9ba131dSTrond Myklebust .mode = 0644, 121d9ba131dSTrond Myklebust .proc_handler = proc_dointvec_minmax, 122d9ba131dSTrond Myklebust .extra1 = &min_slot_table_size, 123d9ba131dSTrond Myklebust .extra2 = &max_tcp_slot_table_limit 124d9ba131dSTrond Myklebust }, 125d9ba131dSTrond Myklebust { 126fbf76683SChuck Lever .procname = "min_resvport", 127fbf76683SChuck Lever .data = &xprt_min_resvport, 128fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 129fbf76683SChuck Lever .mode = 0644, 1306d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 131fbf76683SChuck Lever .extra1 = &xprt_min_resvport_limit, 132e08ea3a9SFrank Sorenson .extra2 = &xprt_max_resvport 133fbf76683SChuck Lever }, 134fbf76683SChuck Lever { 135fbf76683SChuck Lever .procname = "max_resvport", 136fbf76683SChuck Lever .data = &xprt_max_resvport, 137fbf76683SChuck Lever .maxlen = sizeof(unsigned int), 138fbf76683SChuck Lever .mode = 0644, 1396d456111SEric W. Biederman .proc_handler = proc_dointvec_minmax, 140e08ea3a9SFrank Sorenson .extra1 = &xprt_min_resvport, 141fbf76683SChuck Lever .extra2 = &xprt_max_resvport_limit 142fbf76683SChuck Lever }, 143fbf76683SChuck Lever { 14425fe6142STrond Myklebust .procname = "tcp_fin_timeout", 14525fe6142STrond Myklebust .data = &xs_tcp_fin_timeout, 14625fe6142STrond Myklebust .maxlen = sizeof(xs_tcp_fin_timeout), 14725fe6142STrond Myklebust .mode = 0644, 1486d456111SEric W. Biederman .proc_handler = proc_dointvec_jiffies, 14925fe6142STrond Myklebust }, 150f8572d8fSEric W. Biederman { }, 151fbf76683SChuck Lever }; 152fbf76683SChuck Lever 153fe2c6338SJoe Perches static struct ctl_table sunrpc_table[] = { 154fbf76683SChuck Lever { 155fbf76683SChuck Lever .procname = "sunrpc", 156fbf76683SChuck Lever .mode = 0555, 157fbf76683SChuck Lever .child = xs_tunables_table 158fbf76683SChuck Lever }, 159f8572d8fSEric W. Biederman { }, 160fbf76683SChuck Lever }; 161fbf76683SChuck Lever 162fbf76683SChuck Lever #endif 163fbf76683SChuck Lever 164fbf76683SChuck Lever /* 16503bf4b70SChuck Lever * Wait duration for a reply from the RPC portmapper. 16603bf4b70SChuck Lever */ 16703bf4b70SChuck Lever #define XS_BIND_TO (60U * HZ) 16803bf4b70SChuck Lever 16903bf4b70SChuck Lever /* 17003bf4b70SChuck Lever * Delay if a UDP socket connect error occurs. This is most likely some 17103bf4b70SChuck Lever * kind of resource problem on the local host. 17203bf4b70SChuck Lever */ 17303bf4b70SChuck Lever #define XS_UDP_REEST_TO (2U * HZ) 17403bf4b70SChuck Lever 17503bf4b70SChuck Lever /* 17603bf4b70SChuck Lever * The reestablish timeout allows clients to delay for a bit before attempting 17703bf4b70SChuck Lever * to reconnect to a server that just dropped our connection. 17803bf4b70SChuck Lever * 17903bf4b70SChuck Lever * We implement an exponential backoff when trying to reestablish a TCP 18003bf4b70SChuck Lever * transport connection with the server. Some servers like to drop a TCP 18103bf4b70SChuck Lever * connection when they are overworked, so we start with a short timeout and 18203bf4b70SChuck Lever * increase over time if the server is down or not responding. 18303bf4b70SChuck Lever */ 18403bf4b70SChuck Lever #define XS_TCP_INIT_REEST_TO (3U * HZ) 18503bf4b70SChuck Lever 18603bf4b70SChuck Lever /* 18703bf4b70SChuck Lever * TCP idle timeout; client drops the transport socket if it is idle 18803bf4b70SChuck Lever * for this long. Note that we also timeout UDP sockets to prevent 18903bf4b70SChuck Lever * holding port numbers when there is no RPC traffic. 19003bf4b70SChuck Lever */ 19103bf4b70SChuck Lever #define XS_IDLE_DISC_TO (5U * 60 * HZ) 19203bf4b70SChuck Lever 193f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 194a246b010SChuck Lever # undef RPC_DEBUG_DATA 1959903cd1cSChuck Lever # define RPCDBG_FACILITY RPCDBG_TRANS 196a246b010SChuck Lever #endif 197a246b010SChuck Lever 198a246b010SChuck Lever #ifdef RPC_DEBUG_DATA 1999903cd1cSChuck Lever static void xs_pktdump(char *msg, u32 *packet, unsigned int count) 200a246b010SChuck Lever { 201a246b010SChuck Lever u8 *buf = (u8 *) packet; 202a246b010SChuck Lever int j; 203a246b010SChuck Lever 204a246b010SChuck Lever dprintk("RPC: %s\n", msg); 205a246b010SChuck Lever for (j = 0; j < count && j < 128; j += 4) { 206a246b010SChuck Lever if (!(j & 31)) { 207a246b010SChuck Lever if (j) 208a246b010SChuck Lever dprintk("\n"); 209a246b010SChuck Lever dprintk("0x%04x ", j); 210a246b010SChuck Lever } 211a246b010SChuck Lever dprintk("%02x%02x%02x%02x ", 212a246b010SChuck Lever buf[j], buf[j+1], buf[j+2], buf[j+3]); 213a246b010SChuck Lever } 214a246b010SChuck Lever dprintk("\n"); 215a246b010SChuck Lever } 216a246b010SChuck Lever #else 2179903cd1cSChuck Lever static inline void xs_pktdump(char *msg, u32 *packet, unsigned int count) 218a246b010SChuck Lever { 219a246b010SChuck Lever /* NOP */ 220a246b010SChuck Lever } 221a246b010SChuck Lever #endif 222a246b010SChuck Lever 2232118071dSTrond Myklebust static inline struct rpc_xprt *xprt_from_sock(struct sock *sk) 2242118071dSTrond Myklebust { 2252118071dSTrond Myklebust return (struct rpc_xprt *) sk->sk_user_data; 2262118071dSTrond Myklebust } 2272118071dSTrond Myklebust 22895392c59SChuck Lever static inline struct sockaddr *xs_addr(struct rpc_xprt *xprt) 229edb267a6SChuck Lever { 23095392c59SChuck Lever return (struct sockaddr *) &xprt->addr; 23195392c59SChuck Lever } 23295392c59SChuck Lever 233176e21eeSChuck Lever static inline struct sockaddr_un *xs_addr_un(struct rpc_xprt *xprt) 234176e21eeSChuck Lever { 235176e21eeSChuck Lever return (struct sockaddr_un *) &xprt->addr; 236176e21eeSChuck Lever } 237176e21eeSChuck Lever 23895392c59SChuck Lever static inline struct sockaddr_in *xs_addr_in(struct rpc_xprt *xprt) 23995392c59SChuck Lever { 24095392c59SChuck Lever return (struct sockaddr_in *) &xprt->addr; 24195392c59SChuck Lever } 24295392c59SChuck Lever 24395392c59SChuck Lever static inline struct sockaddr_in6 *xs_addr_in6(struct rpc_xprt *xprt) 24495392c59SChuck Lever { 24595392c59SChuck Lever return (struct sockaddr_in6 *) &xprt->addr; 24695392c59SChuck Lever } 24795392c59SChuck Lever 248c877b849SChuck Lever static void xs_format_common_peer_addresses(struct rpc_xprt *xprt) 249c877b849SChuck Lever { 250c877b849SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2519dc3b095SChuck Lever struct sockaddr_in6 *sin6; 2529dc3b095SChuck Lever struct sockaddr_in *sin; 253176e21eeSChuck Lever struct sockaddr_un *sun; 254c877b849SChuck Lever char buf[128]; 255c877b849SChuck Lever 2569dc3b095SChuck Lever switch (sap->sa_family) { 257176e21eeSChuck Lever case AF_LOCAL: 258176e21eeSChuck Lever sun = xs_addr_un(xprt); 259176e21eeSChuck Lever strlcpy(buf, sun->sun_path, sizeof(buf)); 260176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 261176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 262176e21eeSChuck Lever break; 2639dc3b095SChuck Lever case AF_INET: 264176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 265176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 266176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2679dc3b095SChuck Lever sin = xs_addr_in(xprt); 268fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); 2699dc3b095SChuck Lever break; 2709dc3b095SChuck Lever case AF_INET6: 271176e21eeSChuck Lever (void)rpc_ntop(sap, buf, sizeof(buf)); 272176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR] = 273176e21eeSChuck Lever kstrdup(buf, GFP_KERNEL); 2749dc3b095SChuck Lever sin6 = xs_addr_in6(xprt); 275fc0b5791SJoe Perches snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); 2769dc3b095SChuck Lever break; 2779dc3b095SChuck Lever default: 2789dc3b095SChuck Lever BUG(); 2799dc3b095SChuck Lever } 280176e21eeSChuck Lever 2819dc3b095SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); 2829dc3b095SChuck Lever } 2839dc3b095SChuck Lever 2849dc3b095SChuck Lever static void xs_format_common_peer_ports(struct rpc_xprt *xprt) 2859dc3b095SChuck Lever { 2869dc3b095SChuck Lever struct sockaddr *sap = xs_addr(xprt); 2879dc3b095SChuck Lever char buf[128]; 2889dc3b095SChuck Lever 28981160e66SJoe Perches snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); 290c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); 291c877b849SChuck Lever 29281160e66SJoe Perches snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); 293c877b849SChuck Lever xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); 294c877b849SChuck Lever } 295c877b849SChuck Lever 2969dc3b095SChuck Lever static void xs_format_peer_addresses(struct rpc_xprt *xprt, 297b454ae90SChuck Lever const char *protocol, 298b454ae90SChuck Lever const char *netid) 299edb267a6SChuck Lever { 300b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO] = protocol; 301b454ae90SChuck Lever xprt->address_strings[RPC_DISPLAY_NETID] = netid; 302c877b849SChuck Lever xs_format_common_peer_addresses(xprt); 3039dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 304edb267a6SChuck Lever } 305edb267a6SChuck Lever 3069dc3b095SChuck Lever static void xs_update_peer_port(struct rpc_xprt *xprt) 3074b6473fbSChuck Lever { 3089dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); 3099dc3b095SChuck Lever kfree(xprt->address_strings[RPC_DISPLAY_PORT]); 3104b6473fbSChuck Lever 3119dc3b095SChuck Lever xs_format_common_peer_ports(xprt); 312edb267a6SChuck Lever } 313edb267a6SChuck Lever 314edb267a6SChuck Lever static void xs_free_peer_addresses(struct rpc_xprt *xprt) 315edb267a6SChuck Lever { 31633e01dc7SChuck Lever unsigned int i; 31733e01dc7SChuck Lever 31833e01dc7SChuck Lever for (i = 0; i < RPC_DISPLAY_MAX; i++) 31933e01dc7SChuck Lever switch (i) { 32033e01dc7SChuck Lever case RPC_DISPLAY_PROTO: 32133e01dc7SChuck Lever case RPC_DISPLAY_NETID: 32233e01dc7SChuck Lever continue; 32333e01dc7SChuck Lever default: 32433e01dc7SChuck Lever kfree(xprt->address_strings[i]); 32533e01dc7SChuck Lever } 326edb267a6SChuck Lever } 327edb267a6SChuck Lever 328b4b5cc85SChuck Lever #define XS_SENDMSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 329b4b5cc85SChuck Lever 33024c5684bSTrond Myklebust static int xs_send_kvec(struct socket *sock, struct sockaddr *addr, int addrlen, struct kvec *vec, unsigned int base, int more) 331b4b5cc85SChuck Lever { 332b4b5cc85SChuck Lever struct msghdr msg = { 333b4b5cc85SChuck Lever .msg_name = addr, 334b4b5cc85SChuck Lever .msg_namelen = addrlen, 33524c5684bSTrond Myklebust .msg_flags = XS_SENDMSG_FLAGS | (more ? MSG_MORE : 0), 33624c5684bSTrond Myklebust }; 33724c5684bSTrond Myklebust struct kvec iov = { 33824c5684bSTrond Myklebust .iov_base = vec->iov_base + base, 33924c5684bSTrond Myklebust .iov_len = vec->iov_len - base, 340b4b5cc85SChuck Lever }; 341b4b5cc85SChuck Lever 34224c5684bSTrond Myklebust if (iov.iov_len != 0) 343b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, &iov, 1, iov.iov_len); 344b4b5cc85SChuck Lever return kernel_sendmsg(sock, &msg, NULL, 0, 0); 345b4b5cc85SChuck Lever } 346b4b5cc85SChuck Lever 347f279cd00SJason Baron static int xs_send_pagedata(struct socket *sock, struct xdr_buf *xdr, unsigned int base, int more, bool zerocopy, int *sent_p) 348b4b5cc85SChuck Lever { 349a6b31d18STrond Myklebust ssize_t (*do_sendpage)(struct socket *sock, struct page *page, 350a6b31d18STrond Myklebust int offset, size_t size, int flags); 35124c5684bSTrond Myklebust struct page **ppage; 35224c5684bSTrond Myklebust unsigned int remainder; 353f279cd00SJason Baron int err; 354b4b5cc85SChuck Lever 35524c5684bSTrond Myklebust remainder = xdr->page_len - base; 35624c5684bSTrond Myklebust base += xdr->page_base; 35724c5684bSTrond Myklebust ppage = xdr->pages + (base >> PAGE_SHIFT); 35824c5684bSTrond Myklebust base &= ~PAGE_MASK; 359a6b31d18STrond Myklebust do_sendpage = sock->ops->sendpage; 360a6b31d18STrond Myklebust if (!zerocopy) 361a6b31d18STrond Myklebust do_sendpage = sock_no_sendpage; 36224c5684bSTrond Myklebust for(;;) { 36324c5684bSTrond Myklebust unsigned int len = min_t(unsigned int, PAGE_SIZE - base, remainder); 36424c5684bSTrond Myklebust int flags = XS_SENDMSG_FLAGS; 36524c5684bSTrond Myklebust 36624c5684bSTrond Myklebust remainder -= len; 36731303d6cSTrond Myklebust if (more) 36824c5684bSTrond Myklebust flags |= MSG_MORE; 36931303d6cSTrond Myklebust if (remainder != 0) 37031303d6cSTrond Myklebust flags |= MSG_SENDPAGE_NOTLAST | MSG_MORE; 371a6b31d18STrond Myklebust err = do_sendpage(sock, *ppage, base, len, flags); 37224c5684bSTrond Myklebust if (remainder == 0 || err != len) 37324c5684bSTrond Myklebust break; 374f279cd00SJason Baron *sent_p += err; 37524c5684bSTrond Myklebust ppage++; 37624c5684bSTrond Myklebust base = 0; 37724c5684bSTrond Myklebust } 378f279cd00SJason Baron if (err > 0) { 379f279cd00SJason Baron *sent_p += err; 380f279cd00SJason Baron err = 0; 381f279cd00SJason Baron } 38224c5684bSTrond Myklebust return err; 383b4b5cc85SChuck Lever } 384b4b5cc85SChuck Lever 3859903cd1cSChuck Lever /** 3869903cd1cSChuck Lever * xs_sendpages - write pages directly to a socket 3879903cd1cSChuck Lever * @sock: socket to send on 3889903cd1cSChuck Lever * @addr: UDP only -- address of destination 3899903cd1cSChuck Lever * @addrlen: UDP only -- length of destination address 3909903cd1cSChuck Lever * @xdr: buffer containing this request 3919903cd1cSChuck Lever * @base: starting position in the buffer 392a6b31d18STrond Myklebust * @zerocopy: true if it is safe to use sendpage() 393f279cd00SJason Baron * @sent_p: return the total number of bytes successfully queued for sending 3949903cd1cSChuck Lever * 395a246b010SChuck Lever */ 396f279cd00SJason Baron static int xs_sendpages(struct socket *sock, struct sockaddr *addr, int addrlen, struct xdr_buf *xdr, unsigned int base, bool zerocopy, int *sent_p) 397a246b010SChuck Lever { 39824c5684bSTrond Myklebust unsigned int remainder = xdr->len - base; 399f279cd00SJason Baron int err = 0; 400f279cd00SJason Baron int sent = 0; 401a246b010SChuck Lever 402262965f5SChuck Lever if (unlikely(!sock)) 403fba91afbSTrond Myklebust return -ENOTSOCK; 404262965f5SChuck Lever 40524c5684bSTrond Myklebust if (base != 0) { 40624c5684bSTrond Myklebust addr = NULL; 40724c5684bSTrond Myklebust addrlen = 0; 40824c5684bSTrond Myklebust } 409262965f5SChuck Lever 41024c5684bSTrond Myklebust if (base < xdr->head[0].iov_len || addr != NULL) { 41124c5684bSTrond Myklebust unsigned int len = xdr->head[0].iov_len - base; 41224c5684bSTrond Myklebust remainder -= len; 41324c5684bSTrond Myklebust err = xs_send_kvec(sock, addr, addrlen, &xdr->head[0], base, remainder != 0); 41424c5684bSTrond Myklebust if (remainder == 0 || err != len) 415a246b010SChuck Lever goto out; 416f279cd00SJason Baron *sent_p += err; 417a246b010SChuck Lever base = 0; 418a246b010SChuck Lever } else 41924c5684bSTrond Myklebust base -= xdr->head[0].iov_len; 420a246b010SChuck Lever 42124c5684bSTrond Myklebust if (base < xdr->page_len) { 42224c5684bSTrond Myklebust unsigned int len = xdr->page_len - base; 42324c5684bSTrond Myklebust remainder -= len; 424f279cd00SJason Baron err = xs_send_pagedata(sock, xdr, base, remainder != 0, zerocopy, &sent); 425f279cd00SJason Baron *sent_p += sent; 426f279cd00SJason Baron if (remainder == 0 || sent != len) 427a246b010SChuck Lever goto out; 428a246b010SChuck Lever base = 0; 42924c5684bSTrond Myklebust } else 43024c5684bSTrond Myklebust base -= xdr->page_len; 43124c5684bSTrond Myklebust 43224c5684bSTrond Myklebust if (base >= xdr->tail[0].iov_len) 433f279cd00SJason Baron return 0; 43424c5684bSTrond Myklebust err = xs_send_kvec(sock, NULL, 0, &xdr->tail[0], base, 0); 435a246b010SChuck Lever out: 436f279cd00SJason Baron if (err > 0) { 437f279cd00SJason Baron *sent_p += err; 438f279cd00SJason Baron err = 0; 439f279cd00SJason Baron } 44024c5684bSTrond Myklebust return err; 441a246b010SChuck Lever } 442a246b010SChuck Lever 443b6ddf64fSTrond Myklebust static void xs_nospace_callback(struct rpc_task *task) 444b6ddf64fSTrond Myklebust { 445b6ddf64fSTrond Myklebust struct sock_xprt *transport = container_of(task->tk_rqstp->rq_xprt, struct sock_xprt, xprt); 446b6ddf64fSTrond Myklebust 447b6ddf64fSTrond Myklebust transport->inet->sk_write_pending--; 448b6ddf64fSTrond Myklebust } 449b6ddf64fSTrond Myklebust 4509903cd1cSChuck Lever /** 451262965f5SChuck Lever * xs_nospace - place task on wait queue if transmit was incomplete 45250f484e2STrond Myklebust * @req: pointer to RPC request 453262965f5SChuck Lever * @task: task to put to sleep 4549903cd1cSChuck Lever * 455a246b010SChuck Lever */ 45650f484e2STrond Myklebust static int xs_nospace(struct rpc_rqst *req, struct rpc_task *task) 457a246b010SChuck Lever { 458262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 459ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 46006ea0bfeSTrond Myklebust struct sock *sk = transport->inet; 46124ca9a84STrond Myklebust int ret = -EAGAIN; 462a246b010SChuck Lever 46346121cf7SChuck Lever dprintk("RPC: %5u xmit incomplete (%u left of %u)\n", 4646c7a64e5STrond Myklebust task->tk_pid, req->rq_slen - transport->xmit.offset, 465262965f5SChuck Lever req->rq_slen); 466a246b010SChuck Lever 467262965f5SChuck Lever /* Protect against races with write_space */ 468262965f5SChuck Lever spin_lock_bh(&xprt->transport_lock); 469a246b010SChuck Lever 470262965f5SChuck Lever /* Don't race with disconnect */ 471b6ddf64fSTrond Myklebust if (xprt_connected(xprt)) { 47213331a55STrond Myklebust /* wait for more buffer space */ 47306ea0bfeSTrond Myklebust sk->sk_write_pending++; 474b6ddf64fSTrond Myklebust xprt_wait_for_buffer_space(task, xs_nospace_callback); 47513331a55STrond Myklebust } else 4765e3771ceSTrond Myklebust ret = -ENOTCONN; 477a246b010SChuck Lever 478262965f5SChuck Lever spin_unlock_bh(&xprt->transport_lock); 47906ea0bfeSTrond Myklebust 48006ea0bfeSTrond Myklebust /* Race breaker in case memory is freed before above code is called */ 481d48f9ce7SDavid Vrabel if (ret == -EAGAIN) { 482d48f9ce7SDavid Vrabel struct socket_wq *wq; 483d48f9ce7SDavid Vrabel 484d48f9ce7SDavid Vrabel rcu_read_lock(); 485d48f9ce7SDavid Vrabel wq = rcu_dereference(sk->sk_wq); 486d48f9ce7SDavid Vrabel set_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags); 487d48f9ce7SDavid Vrabel rcu_read_unlock(); 488d48f9ce7SDavid Vrabel 48906ea0bfeSTrond Myklebust sk->sk_write_space(sk); 490d48f9ce7SDavid Vrabel } 4915e3771ceSTrond Myklebust return ret; 492a246b010SChuck Lever } 493a246b010SChuck Lever 49461677eeeSChuck Lever /* 4954cd34e7cSTrond Myklebust * Determine if the previous message in the stream was aborted before it 4964cd34e7cSTrond Myklebust * could complete transmission. 4974cd34e7cSTrond Myklebust */ 4984cd34e7cSTrond Myklebust static bool 4994cd34e7cSTrond Myklebust xs_send_request_was_aborted(struct sock_xprt *transport, struct rpc_rqst *req) 5004cd34e7cSTrond Myklebust { 5014cd34e7cSTrond Myklebust return transport->xmit.offset != 0 && req->rq_bytes_sent == 0; 5024cd34e7cSTrond Myklebust } 5034cd34e7cSTrond Myklebust 5044cd34e7cSTrond Myklebust /* 50561677eeeSChuck Lever * Construct a stream transport record marker in @buf. 50661677eeeSChuck Lever */ 50761677eeeSChuck Lever static inline void xs_encode_stream_record_marker(struct xdr_buf *buf) 50861677eeeSChuck Lever { 50961677eeeSChuck Lever u32 reclen = buf->len - sizeof(rpc_fraghdr); 51061677eeeSChuck Lever rpc_fraghdr *base = buf->head[0].iov_base; 51161677eeeSChuck Lever *base = cpu_to_be32(RPC_LAST_STREAM_FRAGMENT | reclen); 51261677eeeSChuck Lever } 51361677eeeSChuck Lever 5149903cd1cSChuck Lever /** 515176e21eeSChuck Lever * xs_local_send_request - write an RPC request to an AF_LOCAL socket 51650f484e2STrond Myklebust * @req: pointer to RPC request 517176e21eeSChuck Lever * @task: RPC task that manages the state of an RPC request 518176e21eeSChuck Lever * 519176e21eeSChuck Lever * Return values: 520176e21eeSChuck Lever * 0: The request has been sent 521176e21eeSChuck Lever * EAGAIN: The socket was blocked, please call again later to 522176e21eeSChuck Lever * complete the request 523176e21eeSChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 524176e21eeSChuck Lever * other: Some other error occured, the request was not sent 525176e21eeSChuck Lever */ 52650f484e2STrond Myklebust static int xs_local_send_request(struct rpc_rqst *req, struct rpc_task *task) 527176e21eeSChuck Lever { 528176e21eeSChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 529176e21eeSChuck Lever struct sock_xprt *transport = 530176e21eeSChuck Lever container_of(xprt, struct sock_xprt, xprt); 531176e21eeSChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 532176e21eeSChuck Lever int status; 533f279cd00SJason Baron int sent = 0; 534176e21eeSChuck Lever 5354cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 5364cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 5374cd34e7cSTrond Myklebust xs_close(xprt); 5384cd34e7cSTrond Myklebust return -ENOTCONN; 5394cd34e7cSTrond Myklebust } 5404cd34e7cSTrond Myklebust 541176e21eeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 542176e21eeSChuck Lever 543176e21eeSChuck Lever xs_pktdump("packet data:", 544176e21eeSChuck Lever req->rq_svec->iov_base, req->rq_svec->iov_len); 545176e21eeSChuck Lever 54678215759SChuck Lever req->rq_xtime = ktime_get(); 5476c7a64e5STrond Myklebust status = xs_sendpages(transport->sock, NULL, 0, xdr, 5486c7a64e5STrond Myklebust transport->xmit.offset, 549f279cd00SJason Baron true, &sent); 550176e21eeSChuck Lever dprintk("RPC: %s(%u) = %d\n", 5516c7a64e5STrond Myklebust __func__, xdr->len - transport->xmit.offset, status); 552743c69e7SNeilBrown 553743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 554743c69e7SNeilBrown status = -ENOBUFS; 555743c69e7SNeilBrown 556f279cd00SJason Baron if (likely(sent > 0) || status == 0) { 5576c7a64e5STrond Myklebust transport->xmit.offset += sent; 5586c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 559176e21eeSChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 5606c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 561176e21eeSChuck Lever req->rq_bytes_sent = 0; 5626c7a64e5STrond Myklebust transport->xmit.offset = 0; 563176e21eeSChuck Lever return 0; 564176e21eeSChuck Lever } 565176e21eeSChuck Lever status = -EAGAIN; 566176e21eeSChuck Lever } 567176e21eeSChuck Lever 568176e21eeSChuck Lever switch (status) { 5693601c4a9STrond Myklebust case -ENOBUFS: 570b5872f0cSTrond Myklebust break; 571176e21eeSChuck Lever case -EAGAIN: 57250f484e2STrond Myklebust status = xs_nospace(req, task); 573176e21eeSChuck Lever break; 574176e21eeSChuck Lever default: 575176e21eeSChuck Lever dprintk("RPC: sendmsg returned unrecognized error %d\n", 576176e21eeSChuck Lever -status); 577e9d47639SGustavo A. R. Silva /* fall through */ 578176e21eeSChuck Lever case -EPIPE: 579176e21eeSChuck Lever xs_close(xprt); 580176e21eeSChuck Lever status = -ENOTCONN; 581176e21eeSChuck Lever } 582176e21eeSChuck Lever 583176e21eeSChuck Lever return status; 584176e21eeSChuck Lever } 585176e21eeSChuck Lever 586176e21eeSChuck Lever /** 587262965f5SChuck Lever * xs_udp_send_request - write an RPC request to a UDP socket 58850f484e2STrond Myklebust * @req: pointer to RPC request 5899903cd1cSChuck Lever * @task: address of RPC task that manages the state of an RPC request 5909903cd1cSChuck Lever * 5919903cd1cSChuck Lever * Return values: 5929903cd1cSChuck Lever * 0: The request has been sent 5939903cd1cSChuck Lever * EAGAIN: The socket was blocked, please call again later to 5949903cd1cSChuck Lever * complete the request 595262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 59625985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 5979903cd1cSChuck Lever */ 59850f484e2STrond Myklebust static int xs_udp_send_request(struct rpc_rqst *req, struct rpc_task *task) 599a246b010SChuck Lever { 600a246b010SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 601ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 602262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 603f279cd00SJason Baron int sent = 0; 604262965f5SChuck Lever int status; 605262965f5SChuck Lever 606262965f5SChuck Lever xs_pktdump("packet data:", 607262965f5SChuck Lever req->rq_svec->iov_base, 608262965f5SChuck Lever req->rq_svec->iov_len); 609262965f5SChuck Lever 61001d37c42STrond Myklebust if (!xprt_bound(xprt)) 61101d37c42STrond Myklebust return -ENOTCONN; 61275891f50STrond Myklebust 61375891f50STrond Myklebust if (!xprt_request_get_cong(xprt, req)) 61475891f50STrond Myklebust return -EBADSLT; 61575891f50STrond Myklebust 61678215759SChuck Lever req->rq_xtime = ktime_get(); 617f279cd00SJason Baron status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen, 6186c7a64e5STrond Myklebust xdr, 0, true, &sent); 619262965f5SChuck Lever 620262965f5SChuck Lever dprintk("RPC: xs_udp_send_request(%u) = %d\n", 6216c7a64e5STrond Myklebust xdr->len, status); 622262965f5SChuck Lever 6233dedbb5cSJason Baron /* firewall is blocking us, don't return -EAGAIN or we end up looping */ 6243dedbb5cSJason Baron if (status == -EPERM) 6253dedbb5cSJason Baron goto process_status; 6263dedbb5cSJason Baron 627743c69e7SNeilBrown if (status == -EAGAIN && sock_writeable(transport->inet)) 628743c69e7SNeilBrown status = -ENOBUFS; 629743c69e7SNeilBrown 630f279cd00SJason Baron if (sent > 0 || status == 0) { 631f279cd00SJason Baron req->rq_xmit_bytes_sent += sent; 632f279cd00SJason Baron if (sent >= req->rq_slen) 633262965f5SChuck Lever return 0; 634262965f5SChuck Lever /* Still some bytes left; set up for a retry later. */ 635262965f5SChuck Lever status = -EAGAIN; 6362199700fSTrond Myklebust } 637262965f5SChuck Lever 6383dedbb5cSJason Baron process_status: 639262965f5SChuck Lever switch (status) { 640fba91afbSTrond Myklebust case -ENOTSOCK: 641fba91afbSTrond Myklebust status = -ENOTCONN; 642fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 643fba91afbSTrond Myklebust break; 644b6ddf64fSTrond Myklebust case -EAGAIN: 64550f484e2STrond Myklebust status = xs_nospace(req, task); 646b6ddf64fSTrond Myklebust break; 647262965f5SChuck Lever case -ENETUNREACH: 6483601c4a9STrond Myklebust case -ENOBUFS: 649262965f5SChuck Lever case -EPIPE: 650262965f5SChuck Lever case -ECONNREFUSED: 6513dedbb5cSJason Baron case -EPERM: 652262965f5SChuck Lever /* When the server has died, an ICMP port unreachable message 653262965f5SChuck Lever * prompts ECONNREFUSED. */ 65413331a55STrond Myklebust break; 65513331a55STrond Myklebust default: 65613331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 65713331a55STrond Myklebust -status); 658262965f5SChuck Lever } 6595fe46e9dSBian Naimeng 660262965f5SChuck Lever return status; 661262965f5SChuck Lever } 662262965f5SChuck Lever 663e06799f9STrond Myklebust /** 664262965f5SChuck Lever * xs_tcp_send_request - write an RPC request to a TCP socket 66550f484e2STrond Myklebust * @req: pointer to RPC request 666262965f5SChuck Lever * @task: address of RPC task that manages the state of an RPC request 667262965f5SChuck Lever * 668262965f5SChuck Lever * Return values: 669262965f5SChuck Lever * 0: The request has been sent 670262965f5SChuck Lever * EAGAIN: The socket was blocked, please call again later to 671262965f5SChuck Lever * complete the request 672262965f5SChuck Lever * ENOTCONN: Caller needs to invoke connect logic then call again 67325985edcSLucas De Marchi * other: Some other error occurred, the request was not sent 674262965f5SChuck Lever * 675262965f5SChuck Lever * XXX: In the case of soft timeouts, should we eventually give up 676262965f5SChuck Lever * if sendmsg is not able to make progress? 677262965f5SChuck Lever */ 67850f484e2STrond Myklebust static int xs_tcp_send_request(struct rpc_rqst *req, struct rpc_task *task) 679262965f5SChuck Lever { 680262965f5SChuck Lever struct rpc_xprt *xprt = req->rq_xprt; 681ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 682262965f5SChuck Lever struct xdr_buf *xdr = &req->rq_snd_buf; 683a6b31d18STrond Myklebust bool zerocopy = true; 6849ffadfbcSTrond Myklebust bool vm_wait = false; 685b595bb15SChuck Lever int status; 686f279cd00SJason Baron int sent; 687a246b010SChuck Lever 6884cd34e7cSTrond Myklebust /* Close the stream if the previous transmission was incomplete */ 6894cd34e7cSTrond Myklebust if (xs_send_request_was_aborted(transport, req)) { 6904cd34e7cSTrond Myklebust if (transport->sock != NULL) 6914cd34e7cSTrond Myklebust kernel_sock_shutdown(transport->sock, SHUT_RDWR); 6924cd34e7cSTrond Myklebust return -ENOTCONN; 6934cd34e7cSTrond Myklebust } 6944cd34e7cSTrond Myklebust 69561677eeeSChuck Lever xs_encode_stream_record_marker(&req->rq_snd_buf); 696262965f5SChuck Lever 697262965f5SChuck Lever xs_pktdump("packet data:", 698262965f5SChuck Lever req->rq_svec->iov_base, 699262965f5SChuck Lever req->rq_svec->iov_len); 700a6b31d18STrond Myklebust /* Don't use zero copy if this is a resend. If the RPC call 701a6b31d18STrond Myklebust * completes while the socket holds a reference to the pages, 702a6b31d18STrond Myklebust * then we may end up resending corrupted data. 703a6b31d18STrond Myklebust */ 70450f484e2STrond Myklebust if (req->rq_task->tk_flags & RPC_TASK_SENT) 705a6b31d18STrond Myklebust zerocopy = false; 706a246b010SChuck Lever 7077196dbb0STrond Myklebust if (test_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state)) 7087196dbb0STrond Myklebust xs_tcp_set_socket_timeouts(xprt, transport->sock); 7097196dbb0STrond Myklebust 710a246b010SChuck Lever /* Continue transmitting the packet/record. We must be careful 711a246b010SChuck Lever * to cope with writespace callbacks arriving _after_ we have 712262965f5SChuck Lever * called sendmsg(). */ 71378215759SChuck Lever req->rq_xtime = ktime_get(); 714a246b010SChuck Lever while (1) { 715f279cd00SJason Baron sent = 0; 716f279cd00SJason Baron status = xs_sendpages(transport->sock, NULL, 0, xdr, 7176c7a64e5STrond Myklebust transport->xmit.offset, 7186c7a64e5STrond Myklebust zerocopy, &sent); 719a246b010SChuck Lever 720262965f5SChuck Lever dprintk("RPC: xs_tcp_send_request(%u) = %d\n", 7216c7a64e5STrond Myklebust xdr->len - transport->xmit.offset, status); 722262965f5SChuck Lever 723a246b010SChuck Lever /* If we've sent the entire packet, immediately 724a246b010SChuck Lever * reset the count of bytes sent. */ 7256c7a64e5STrond Myklebust transport->xmit.offset += sent; 7266c7a64e5STrond Myklebust req->rq_bytes_sent = transport->xmit.offset; 727262965f5SChuck Lever if (likely(req->rq_bytes_sent >= req->rq_slen)) { 7286c7a64e5STrond Myklebust req->rq_xmit_bytes_sent += transport->xmit.offset; 729a246b010SChuck Lever req->rq_bytes_sent = 0; 7306c7a64e5STrond Myklebust transport->xmit.offset = 0; 731a246b010SChuck Lever return 0; 732a246b010SChuck Lever } 733262965f5SChuck Lever 7349ffadfbcSTrond Myklebust WARN_ON_ONCE(sent == 0 && status == 0); 7359ffadfbcSTrond Myklebust 7369ffadfbcSTrond Myklebust if (status == -EAGAIN ) { 7379ffadfbcSTrond Myklebust /* 7389ffadfbcSTrond Myklebust * Return EAGAIN if we're sure we're hitting the 7399ffadfbcSTrond Myklebust * socket send buffer limits. 7409ffadfbcSTrond Myklebust */ 7419ffadfbcSTrond Myklebust if (test_bit(SOCK_NOSPACE, &transport->sock->flags)) 7429ffadfbcSTrond Myklebust break; 7439ffadfbcSTrond Myklebust /* 7449ffadfbcSTrond Myklebust * Did we hit a memory allocation failure? 7459ffadfbcSTrond Myklebust */ 7469ffadfbcSTrond Myklebust if (sent == 0) { 7479ffadfbcSTrond Myklebust status = -ENOBUFS; 7489ffadfbcSTrond Myklebust if (vm_wait) 7499ffadfbcSTrond Myklebust break; 7509ffadfbcSTrond Myklebust /* Retry, knowing now that we're below the 7519ffadfbcSTrond Myklebust * socket send buffer limit 7529ffadfbcSTrond Myklebust */ 7539ffadfbcSTrond Myklebust vm_wait = true; 7549ffadfbcSTrond Myklebust } 7559ffadfbcSTrond Myklebust continue; 7569ffadfbcSTrond Myklebust } 757f580dd04STrond Myklebust if (status < 0) 758f580dd04STrond Myklebust break; 7599ffadfbcSTrond Myklebust vm_wait = false; 760a246b010SChuck Lever } 761a246b010SChuck Lever 762262965f5SChuck Lever switch (status) { 763fba91afbSTrond Myklebust case -ENOTSOCK: 764fba91afbSTrond Myklebust status = -ENOTCONN; 765fba91afbSTrond Myklebust /* Should we call xs_close() here? */ 766fba91afbSTrond Myklebust break; 767262965f5SChuck Lever case -EAGAIN: 76850f484e2STrond Myklebust status = xs_nospace(req, task); 769262965f5SChuck Lever break; 770262965f5SChuck Lever case -ECONNRESET: 771262965f5SChuck Lever case -ECONNREFUSED: 772262965f5SChuck Lever case -ENOTCONN: 7733913c78cSTrond Myklebust case -EADDRINUSE: 774b5872f0cSTrond Myklebust case -ENOBUFS: 775b9d2bb2eSTrond Myklebust case -EPIPE: 77613331a55STrond Myklebust break; 77713331a55STrond Myklebust default: 77813331a55STrond Myklebust dprintk("RPC: sendmsg returned unrecognized error %d\n", 77913331a55STrond Myklebust -status); 780a246b010SChuck Lever } 7815fe46e9dSBian Naimeng 782a246b010SChuck Lever return status; 783a246b010SChuck Lever } 784a246b010SChuck Lever 7852a9e1cfaSTrond Myklebust static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) 7862a9e1cfaSTrond Myklebust { 7872a9e1cfaSTrond Myklebust transport->old_data_ready = sk->sk_data_ready; 7882a9e1cfaSTrond Myklebust transport->old_state_change = sk->sk_state_change; 7892a9e1cfaSTrond Myklebust transport->old_write_space = sk->sk_write_space; 7902118071dSTrond Myklebust transport->old_error_report = sk->sk_error_report; 7912a9e1cfaSTrond Myklebust } 7922a9e1cfaSTrond Myklebust 7932a9e1cfaSTrond Myklebust static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) 7942a9e1cfaSTrond Myklebust { 7952a9e1cfaSTrond Myklebust sk->sk_data_ready = transport->old_data_ready; 7962a9e1cfaSTrond Myklebust sk->sk_state_change = transport->old_state_change; 7972a9e1cfaSTrond Myklebust sk->sk_write_space = transport->old_write_space; 7982118071dSTrond Myklebust sk->sk_error_report = transport->old_error_report; 7992118071dSTrond Myklebust } 8002118071dSTrond Myklebust 80142d42a5bSTrond Myklebust static void xs_sock_reset_state_flags(struct rpc_xprt *xprt) 80242d42a5bSTrond Myklebust { 80342d42a5bSTrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 80442d42a5bSTrond Myklebust 80542d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 80642d42a5bSTrond Myklebust } 80742d42a5bSTrond Myklebust 808b70ae915STrond Myklebust static void xs_sock_reset_connection_flags(struct rpc_xprt *xprt) 809b70ae915STrond Myklebust { 810b70ae915STrond Myklebust smp_mb__before_atomic(); 811b70ae915STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 812b70ae915STrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 81342d42a5bSTrond Myklebust xs_sock_reset_state_flags(xprt); 814b70ae915STrond Myklebust smp_mb__after_atomic(); 815b70ae915STrond Myklebust } 816b70ae915STrond Myklebust 8172118071dSTrond Myklebust /** 8182118071dSTrond Myklebust * xs_error_report - callback to handle TCP socket state errors 8192118071dSTrond Myklebust * @sk: socket 8202118071dSTrond Myklebust * 8212118071dSTrond Myklebust * Note: we don't call sock_error() since there may be a rpc_task 8222118071dSTrond Myklebust * using the socket, and so we don't want to clear sk->sk_err. 8232118071dSTrond Myklebust */ 8242118071dSTrond Myklebust static void xs_error_report(struct sock *sk) 8252118071dSTrond Myklebust { 8262118071dSTrond Myklebust struct rpc_xprt *xprt; 8272118071dSTrond Myklebust int err; 8282118071dSTrond Myklebust 8292118071dSTrond Myklebust read_lock_bh(&sk->sk_callback_lock); 8302118071dSTrond Myklebust if (!(xprt = xprt_from_sock(sk))) 8312118071dSTrond Myklebust goto out; 8322118071dSTrond Myklebust 8332118071dSTrond Myklebust err = -sk->sk_err; 8342118071dSTrond Myklebust if (err == 0) 8352118071dSTrond Myklebust goto out; 8362118071dSTrond Myklebust dprintk("RPC: xs_error_report client %p, error=%d...\n", 8372118071dSTrond Myklebust xprt, -err); 838e8353c76STrond Myklebust trace_rpc_socket_error(xprt, sk->sk_socket, err); 8392118071dSTrond Myklebust xprt_wake_pending_tasks(xprt, err); 8402118071dSTrond Myklebust out: 8412118071dSTrond Myklebust read_unlock_bh(&sk->sk_callback_lock); 8422a9e1cfaSTrond Myklebust } 8432a9e1cfaSTrond Myklebust 844fe315e76SChuck Lever static void xs_reset_transport(struct sock_xprt *transport) 845a246b010SChuck Lever { 846ee0ac0c2SChuck Lever struct socket *sock = transport->sock; 847ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 8486cc7e908STrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 849a246b010SChuck Lever 850fe315e76SChuck Lever if (sk == NULL) 851fe315e76SChuck Lever return; 8529903cd1cSChuck Lever 853264d1df3SJeff Layton if (atomic_read(&transport->xprt.swapper)) 854264d1df3SJeff Layton sk_clear_memalloc(sk); 855264d1df3SJeff Layton 85609939204STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 85709939204STrond Myklebust 858edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 859a246b010SChuck Lever write_lock_bh(&sk->sk_callback_lock); 860ee0ac0c2SChuck Lever transport->inet = NULL; 861ee0ac0c2SChuck Lever transport->sock = NULL; 862a246b010SChuck Lever 863a246b010SChuck Lever sk->sk_user_data = NULL; 8642a9e1cfaSTrond Myklebust 8652a9e1cfaSTrond Myklebust xs_restore_old_callbacks(transport, sk); 8660c78789eSTrond Myklebust xprt_clear_connected(xprt); 867a246b010SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 8686cc7e908STrond Myklebust xs_sock_reset_connection_flags(xprt); 869edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 870a246b010SChuck Lever 8716cc7e908STrond Myklebust trace_rpc_socket_close(xprt, sock); 872a246b010SChuck Lever sock_release(sock); 873fe315e76SChuck Lever } 874fe315e76SChuck Lever 875fe315e76SChuck Lever /** 876fe315e76SChuck Lever * xs_close - close a socket 877fe315e76SChuck Lever * @xprt: transport 878fe315e76SChuck Lever * 879fe315e76SChuck Lever * This is used when all requests are complete; ie, no DRC state remains 880fe315e76SChuck Lever * on the server we want to save. 881f75e6745STrond Myklebust * 882f75e6745STrond Myklebust * The caller _must_ be holding XPRT_LOCKED in order to avoid issues with 883f75e6745STrond Myklebust * xs_reset_transport() zeroing the socket from underneath a writer. 884fe315e76SChuck Lever */ 885fe315e76SChuck Lever static void xs_close(struct rpc_xprt *xprt) 886fe315e76SChuck Lever { 887fe315e76SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 888fe315e76SChuck Lever 889fe315e76SChuck Lever dprintk("RPC: xs_close xprt %p\n", xprt); 890fe315e76SChuck Lever 891fe315e76SChuck Lever xs_reset_transport(transport); 89261d0a8e6SNeil Brown xprt->reestablish_timeout = 0; 893fe315e76SChuck Lever 89462da3b24STrond Myklebust xprt_disconnect_done(xprt); 895a246b010SChuck Lever } 896a246b010SChuck Lever 8974a068258SChuck Lever static void xs_inject_disconnect(struct rpc_xprt *xprt) 8984a068258SChuck Lever { 8994a068258SChuck Lever dprintk("RPC: injecting transport disconnect on xprt=%p\n", 9004a068258SChuck Lever xprt); 9014a068258SChuck Lever xprt_disconnect_done(xprt); 9024a068258SChuck Lever } 9034a068258SChuck Lever 904315f3812SKinglong Mee static void xs_xprt_free(struct rpc_xprt *xprt) 905315f3812SKinglong Mee { 906315f3812SKinglong Mee xs_free_peer_addresses(xprt); 907315f3812SKinglong Mee xprt_free(xprt); 908315f3812SKinglong Mee } 909315f3812SKinglong Mee 9109903cd1cSChuck Lever /** 9119903cd1cSChuck Lever * xs_destroy - prepare to shutdown a transport 9129903cd1cSChuck Lever * @xprt: doomed transport 9139903cd1cSChuck Lever * 9149903cd1cSChuck Lever */ 9159903cd1cSChuck Lever static void xs_destroy(struct rpc_xprt *xprt) 916a246b010SChuck Lever { 91703c78827STrond Myklebust struct sock_xprt *transport = container_of(xprt, 91803c78827STrond Myklebust struct sock_xprt, xprt); 9199903cd1cSChuck Lever dprintk("RPC: xs_destroy xprt %p\n", xprt); 9209903cd1cSChuck Lever 92103c78827STrond Myklebust cancel_delayed_work_sync(&transport->connect_worker); 922a1311d87STrond Myklebust xs_close(xprt); 923edc1b01cSTrond Myklebust cancel_work_sync(&transport->recv_worker); 924315f3812SKinglong Mee xs_xprt_free(xprt); 925a1311d87STrond Myklebust module_put(THIS_MODULE); 926a246b010SChuck Lever } 927a246b010SChuck Lever 928176e21eeSChuck Lever static int xs_local_copy_to_xdr(struct xdr_buf *xdr, struct sk_buff *skb) 929176e21eeSChuck Lever { 930176e21eeSChuck Lever struct xdr_skb_reader desc = { 931176e21eeSChuck Lever .skb = skb, 932176e21eeSChuck Lever .offset = sizeof(rpc_fraghdr), 933176e21eeSChuck Lever .count = skb->len - sizeof(rpc_fraghdr), 934176e21eeSChuck Lever }; 935176e21eeSChuck Lever 936176e21eeSChuck Lever if (xdr_partial_copy_from_skb(xdr, 0, &desc, xdr_skb_read_bits) < 0) 937176e21eeSChuck Lever return -1; 938176e21eeSChuck Lever if (desc.count) 939176e21eeSChuck Lever return -1; 940176e21eeSChuck Lever return 0; 941176e21eeSChuck Lever } 942176e21eeSChuck Lever 943176e21eeSChuck Lever /** 944a2648094STrond Myklebust * xs_local_data_read_skb 945a2648094STrond Myklebust * @xprt: transport 946a2648094STrond Myklebust * @sk: socket 947a2648094STrond Myklebust * @skb: skbuff 948176e21eeSChuck Lever * 949176e21eeSChuck Lever * Currently this assumes we can read the whole reply in a single gulp. 950176e21eeSChuck Lever */ 951a2648094STrond Myklebust static void xs_local_data_read_skb(struct rpc_xprt *xprt, 952a2648094STrond Myklebust struct sock *sk, 953a2648094STrond Myklebust struct sk_buff *skb) 954176e21eeSChuck Lever { 955176e21eeSChuck Lever struct rpc_task *task; 956176e21eeSChuck Lever struct rpc_rqst *rovr; 957a2648094STrond Myklebust int repsize, copied; 958176e21eeSChuck Lever u32 _xid; 959176e21eeSChuck Lever __be32 *xp; 960176e21eeSChuck Lever 961176e21eeSChuck Lever repsize = skb->len - sizeof(rpc_fraghdr); 962176e21eeSChuck Lever if (repsize < 4) { 963176e21eeSChuck Lever dprintk("RPC: impossible RPC reply size %d\n", repsize); 964a2648094STrond Myklebust return; 965176e21eeSChuck Lever } 966176e21eeSChuck Lever 967176e21eeSChuck Lever /* Copy the XID from the skb... */ 968176e21eeSChuck Lever xp = skb_header_pointer(skb, sizeof(rpc_fraghdr), sizeof(_xid), &_xid); 969176e21eeSChuck Lever if (xp == NULL) 970a2648094STrond Myklebust return; 971176e21eeSChuck Lever 972176e21eeSChuck Lever /* Look up and lock the request corresponding to the given XID */ 97375c84151STrond Myklebust spin_lock(&xprt->queue_lock); 974176e21eeSChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 975176e21eeSChuck Lever if (!rovr) 976176e21eeSChuck Lever goto out_unlock; 977729749bbSTrond Myklebust xprt_pin_rqst(rovr); 97875c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 979176e21eeSChuck Lever task = rovr->rq_task; 980176e21eeSChuck Lever 981176e21eeSChuck Lever copied = rovr->rq_private_buf.buflen; 982176e21eeSChuck Lever if (copied > repsize) 983176e21eeSChuck Lever copied = repsize; 984176e21eeSChuck Lever 985176e21eeSChuck Lever if (xs_local_copy_to_xdr(&rovr->rq_private_buf, skb)) { 986176e21eeSChuck Lever dprintk("RPC: sk_buff copy failed\n"); 98775c84151STrond Myklebust spin_lock(&xprt->queue_lock); 988729749bbSTrond Myklebust goto out_unpin; 989176e21eeSChuck Lever } 990176e21eeSChuck Lever 99175c84151STrond Myklebust spin_lock(&xprt->queue_lock); 992176e21eeSChuck Lever xprt_complete_rqst(task, copied); 993729749bbSTrond Myklebust out_unpin: 994729749bbSTrond Myklebust xprt_unpin_rqst(rovr); 995176e21eeSChuck Lever out_unlock: 99675c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 997a2648094STrond Myklebust } 998a2648094STrond Myklebust 999a2648094STrond Myklebust static void xs_local_data_receive(struct sock_xprt *transport) 1000a2648094STrond Myklebust { 1001a2648094STrond Myklebust struct sk_buff *skb; 1002a2648094STrond Myklebust struct sock *sk; 1003a2648094STrond Myklebust int err; 1004a2648094STrond Myklebust 10050af3442aSTrond Myklebust restart: 1006a2648094STrond Myklebust mutex_lock(&transport->recv_mutex); 1007a2648094STrond Myklebust sk = transport->inet; 1008a2648094STrond Myklebust if (sk == NULL) 1009a2648094STrond Myklebust goto out; 1010a2648094STrond Myklebust for (;;) { 1011a2648094STrond Myklebust skb = skb_recv_datagram(sk, 0, 1, &err); 101242d42a5bSTrond Myklebust if (skb != NULL) { 1013a2648094STrond Myklebust xs_local_data_read_skb(&transport->xprt, sk, skb); 1014176e21eeSChuck Lever skb_free_datagram(sk, skb); 101542d42a5bSTrond Myklebust continue; 101642d42a5bSTrond Myklebust } 101742d42a5bSTrond Myklebust if (!test_and_clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 101842d42a5bSTrond Myklebust break; 10190af3442aSTrond Myklebust if (need_resched()) { 10200af3442aSTrond Myklebust mutex_unlock(&transport->recv_mutex); 10210af3442aSTrond Myklebust cond_resched(); 10220af3442aSTrond Myklebust goto restart; 10230af3442aSTrond Myklebust } 1024a2648094STrond Myklebust } 1025176e21eeSChuck Lever out: 1026a2648094STrond Myklebust mutex_unlock(&transport->recv_mutex); 1027a2648094STrond Myklebust } 1028a2648094STrond Myklebust 1029a2648094STrond Myklebust static void xs_local_data_receive_workfn(struct work_struct *work) 1030a2648094STrond Myklebust { 1031a2648094STrond Myklebust struct sock_xprt *transport = 1032a2648094STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1033a2648094STrond Myklebust xs_local_data_receive(transport); 1034176e21eeSChuck Lever } 1035176e21eeSChuck Lever 10369903cd1cSChuck Lever /** 1037f9b2ee71STrond Myklebust * xs_udp_data_read_skb - receive callback for UDP sockets 1038f9b2ee71STrond Myklebust * @xprt: transport 1039f9b2ee71STrond Myklebust * @sk: socket 1040f9b2ee71STrond Myklebust * @skb: skbuff 10419903cd1cSChuck Lever * 1042a246b010SChuck Lever */ 1043f9b2ee71STrond Myklebust static void xs_udp_data_read_skb(struct rpc_xprt *xprt, 1044f9b2ee71STrond Myklebust struct sock *sk, 1045f9b2ee71STrond Myklebust struct sk_buff *skb) 1046a246b010SChuck Lever { 1047a246b010SChuck Lever struct rpc_task *task; 1048a246b010SChuck Lever struct rpc_rqst *rovr; 1049f9b2ee71STrond Myklebust int repsize, copied; 1050d8ed029dSAlexey Dobriyan u32 _xid; 1051d8ed029dSAlexey Dobriyan __be32 *xp; 1052a246b010SChuck Lever 10531da8c681SWillem de Bruijn repsize = skb->len; 1054a246b010SChuck Lever if (repsize < 4) { 10559903cd1cSChuck Lever dprintk("RPC: impossible RPC reply size %d!\n", repsize); 1056f9b2ee71STrond Myklebust return; 1057a246b010SChuck Lever } 1058a246b010SChuck Lever 1059a246b010SChuck Lever /* Copy the XID from the skb... */ 10601da8c681SWillem de Bruijn xp = skb_header_pointer(skb, 0, sizeof(_xid), &_xid); 1061a246b010SChuck Lever if (xp == NULL) 1062f9b2ee71STrond Myklebust return; 1063a246b010SChuck Lever 1064a246b010SChuck Lever /* Look up and lock the request corresponding to the given XID */ 106575c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1066a246b010SChuck Lever rovr = xprt_lookup_rqst(xprt, *xp); 1067a246b010SChuck Lever if (!rovr) 1068a246b010SChuck Lever goto out_unlock; 1069729749bbSTrond Myklebust xprt_pin_rqst(rovr); 1070ecd465eeSChuck Lever xprt_update_rtt(rovr->rq_task); 107175c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1072a246b010SChuck Lever task = rovr->rq_task; 1073a246b010SChuck Lever 1074a246b010SChuck Lever if ((copied = rovr->rq_private_buf.buflen) > repsize) 1075a246b010SChuck Lever copied = repsize; 1076a246b010SChuck Lever 1077a246b010SChuck Lever /* Suck it into the iovec, verify checksum if not done by hw. */ 10781781f7f5SHerbert Xu if (csum_partial_copy_to_xdr(&rovr->rq_private_buf, skb)) { 107975c84151STrond Myklebust spin_lock(&xprt->queue_lock); 10800afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INERRORS); 1081729749bbSTrond Myklebust goto out_unpin; 10821781f7f5SHerbert Xu } 10831781f7f5SHerbert Xu 1084a246b010SChuck Lever 1085729749bbSTrond Myklebust spin_lock_bh(&xprt->transport_lock); 10866a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, copied); 1087ce7c252aSTrond Myklebust spin_unlock_bh(&xprt->transport_lock); 108875c84151STrond Myklebust spin_lock(&xprt->queue_lock); 10891570c1e4SChuck Lever xprt_complete_rqst(task, copied); 10900afa6b44STrond Myklebust __UDPX_INC_STATS(sk, UDP_MIB_INDATAGRAMS); 1091729749bbSTrond Myklebust out_unpin: 1092729749bbSTrond Myklebust xprt_unpin_rqst(rovr); 1093a246b010SChuck Lever out_unlock: 109475c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 1095f9b2ee71STrond Myklebust } 1096f9b2ee71STrond Myklebust 1097f9b2ee71STrond Myklebust static void xs_udp_data_receive(struct sock_xprt *transport) 1098f9b2ee71STrond Myklebust { 1099f9b2ee71STrond Myklebust struct sk_buff *skb; 1100f9b2ee71STrond Myklebust struct sock *sk; 1101f9b2ee71STrond Myklebust int err; 1102f9b2ee71STrond Myklebust 11030af3442aSTrond Myklebust restart: 1104f9b2ee71STrond Myklebust mutex_lock(&transport->recv_mutex); 1105f9b2ee71STrond Myklebust sk = transport->inet; 1106f9b2ee71STrond Myklebust if (sk == NULL) 1107f9b2ee71STrond Myklebust goto out; 1108f9b2ee71STrond Myklebust for (;;) { 11097c13f97fSPaolo Abeni skb = skb_recv_udp(sk, 0, 1, &err); 111042d42a5bSTrond Myklebust if (skb != NULL) { 1111f9b2ee71STrond Myklebust xs_udp_data_read_skb(&transport->xprt, sk, skb); 1112850cbaddSPaolo Abeni consume_skb(skb); 111342d42a5bSTrond Myklebust continue; 111442d42a5bSTrond Myklebust } 111542d42a5bSTrond Myklebust if (!test_and_clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 111642d42a5bSTrond Myklebust break; 11170af3442aSTrond Myklebust if (need_resched()) { 11180af3442aSTrond Myklebust mutex_unlock(&transport->recv_mutex); 11190af3442aSTrond Myklebust cond_resched(); 11200af3442aSTrond Myklebust goto restart; 11210af3442aSTrond Myklebust } 1122f9b2ee71STrond Myklebust } 1123a246b010SChuck Lever out: 1124f9b2ee71STrond Myklebust mutex_unlock(&transport->recv_mutex); 1125f9b2ee71STrond Myklebust } 1126f9b2ee71STrond Myklebust 1127f9b2ee71STrond Myklebust static void xs_udp_data_receive_workfn(struct work_struct *work) 1128f9b2ee71STrond Myklebust { 1129f9b2ee71STrond Myklebust struct sock_xprt *transport = 1130f9b2ee71STrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1131f9b2ee71STrond Myklebust xs_udp_data_receive(transport); 1132f9b2ee71STrond Myklebust } 1133f9b2ee71STrond Myklebust 1134f9b2ee71STrond Myklebust /** 1135f9b2ee71STrond Myklebust * xs_data_ready - "data ready" callback for UDP sockets 1136f9b2ee71STrond Myklebust * @sk: socket with data to read 1137f9b2ee71STrond Myklebust * 1138f9b2ee71STrond Myklebust */ 1139f9b2ee71STrond Myklebust static void xs_data_ready(struct sock *sk) 1140f9b2ee71STrond Myklebust { 1141f9b2ee71STrond Myklebust struct rpc_xprt *xprt; 1142f9b2ee71STrond Myklebust 1143f9b2ee71STrond Myklebust read_lock_bh(&sk->sk_callback_lock); 1144f9b2ee71STrond Myklebust dprintk("RPC: xs_data_ready...\n"); 1145f9b2ee71STrond Myklebust xprt = xprt_from_sock(sk); 1146f9b2ee71STrond Myklebust if (xprt != NULL) { 1147f9b2ee71STrond Myklebust struct sock_xprt *transport = container_of(xprt, 1148f9b2ee71STrond Myklebust struct sock_xprt, xprt); 11495157b956STrond Myklebust transport->old_data_ready(sk); 11505157b956STrond Myklebust /* Any data means we had a useful conversation, so 11515157b956STrond Myklebust * then we don't need to delay the next reconnect 11525157b956STrond Myklebust */ 11535157b956STrond Myklebust if (xprt->reestablish_timeout) 11545157b956STrond Myklebust xprt->reestablish_timeout = 0; 115542d42a5bSTrond Myklebust if (!test_and_set_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 115640a5f1b1STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 1157f9b2ee71STrond Myklebust } 1158f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1159a246b010SChuck Lever } 1160a246b010SChuck Lever 1161a519fc7aSTrond Myklebust /* 1162a519fc7aSTrond Myklebust * Helper function to force a TCP close if the server is sending 1163a519fc7aSTrond Myklebust * junk and/or it has put us in CLOSE_WAIT 1164a519fc7aSTrond Myklebust */ 1165a519fc7aSTrond Myklebust static void xs_tcp_force_close(struct rpc_xprt *xprt) 1166a519fc7aSTrond Myklebust { 1167a519fc7aSTrond Myklebust xprt_force_disconnect(xprt); 1168a519fc7aSTrond Myklebust } 1169a519fc7aSTrond Myklebust 1170dd456471SChuck Lever static inline void xs_tcp_read_fraghdr(struct rpc_xprt *xprt, struct xdr_skb_reader *desc) 1171a246b010SChuck Lever { 117251971139SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1173a246b010SChuck Lever size_t len, used; 1174a246b010SChuck Lever char *p; 1175a246b010SChuck Lever 1176d1109aa5STrond Myklebust p = ((char *) &transport->recv.fraghdr) + transport->recv.offset; 1177d1109aa5STrond Myklebust len = sizeof(transport->recv.fraghdr) - transport->recv.offset; 11789d292316SChuck Lever used = xdr_skb_read_bits(desc, p, len); 1179d1109aa5STrond Myklebust transport->recv.offset += used; 1180a246b010SChuck Lever if (used != len) 1181a246b010SChuck Lever return; 1182808012fbSChuck Lever 1183d1109aa5STrond Myklebust transport->recv.len = ntohl(transport->recv.fraghdr); 1184d1109aa5STrond Myklebust if (transport->recv.len & RPC_LAST_STREAM_FRAGMENT) 1185d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_LAST_FRAG; 1186a246b010SChuck Lever else 1187d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_LAST_FRAG; 1188d1109aa5STrond Myklebust transport->recv.len &= RPC_FRAGMENT_SIZE_MASK; 1189808012fbSChuck Lever 1190d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_FRAGHDR; 1191d1109aa5STrond Myklebust transport->recv.offset = 0; 1192808012fbSChuck Lever 1193a246b010SChuck Lever /* Sanity check of the record length */ 1194d1109aa5STrond Myklebust if (unlikely(transport->recv.len < 8)) { 11959903cd1cSChuck Lever dprintk("RPC: invalid TCP record fragment length\n"); 1196a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 11979903cd1cSChuck Lever return; 1198a246b010SChuck Lever } 1199a246b010SChuck Lever dprintk("RPC: reading TCP record fragment of length %d\n", 1200d1109aa5STrond Myklebust transport->recv.len); 1201a246b010SChuck Lever } 1202a246b010SChuck Lever 120351971139SChuck Lever static void xs_tcp_check_fraghdr(struct sock_xprt *transport) 1204a246b010SChuck Lever { 1205d1109aa5STrond Myklebust if (transport->recv.offset == transport->recv.len) { 1206d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_FRAGHDR; 1207d1109aa5STrond Myklebust transport->recv.offset = 0; 1208d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_LAST_FRAG) { 1209d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_DATA; 1210d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_XID; 1211d1109aa5STrond Myklebust transport->recv.copied = 0; 1212a246b010SChuck Lever } 1213a246b010SChuck Lever } 1214a246b010SChuck Lever } 1215a246b010SChuck Lever 1216dd456471SChuck Lever static inline void xs_tcp_read_xid(struct sock_xprt *transport, struct xdr_skb_reader *desc) 1217a246b010SChuck Lever { 1218a246b010SChuck Lever size_t len, used; 1219a246b010SChuck Lever char *p; 1220a246b010SChuck Lever 1221d1109aa5STrond Myklebust len = sizeof(transport->recv.xid) - transport->recv.offset; 12225b5e0928SAlexey Dobriyan dprintk("RPC: reading XID (%zu bytes)\n", len); 1223d1109aa5STrond Myklebust p = ((char *) &transport->recv.xid) + transport->recv.offset; 12249d292316SChuck Lever used = xdr_skb_read_bits(desc, p, len); 1225d1109aa5STrond Myklebust transport->recv.offset += used; 1226a246b010SChuck Lever if (used != len) 1227a246b010SChuck Lever return; 1228d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_XID; 1229d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_READ_CALLDIR; 1230d1109aa5STrond Myklebust transport->recv.copied = 4; 123118dca02aSRicardo Labiaga dprintk("RPC: reading %s XID %08x\n", 1232d1109aa5STrond Myklebust (transport->recv.flags & TCP_RPC_REPLY) ? "reply for" 123318dca02aSRicardo Labiaga : "request with", 1234d1109aa5STrond Myklebust ntohl(transport->recv.xid)); 123551971139SChuck Lever xs_tcp_check_fraghdr(transport); 1236a246b010SChuck Lever } 1237a246b010SChuck Lever 123818dca02aSRicardo Labiaga static inline void xs_tcp_read_calldir(struct sock_xprt *transport, 123918dca02aSRicardo Labiaga struct xdr_skb_reader *desc) 1240a246b010SChuck Lever { 124118dca02aSRicardo Labiaga size_t len, used; 124218dca02aSRicardo Labiaga u32 offset; 1243b76ce561STrond Myklebust char *p; 124418dca02aSRicardo Labiaga 124518dca02aSRicardo Labiaga /* 1246d1109aa5STrond Myklebust * We want transport->recv.offset to be 8 at the end of this routine 124718dca02aSRicardo Labiaga * (4 bytes for the xid and 4 bytes for the call/reply flag). 124818dca02aSRicardo Labiaga * When this function is called for the first time, 1249d1109aa5STrond Myklebust * transport->recv.offset is 4 (after having already read the xid). 125018dca02aSRicardo Labiaga */ 1251d1109aa5STrond Myklebust offset = transport->recv.offset - sizeof(transport->recv.xid); 1252d1109aa5STrond Myklebust len = sizeof(transport->recv.calldir) - offset; 12535b5e0928SAlexey Dobriyan dprintk("RPC: reading CALL/REPLY flag (%zu bytes)\n", len); 1254d1109aa5STrond Myklebust p = ((char *) &transport->recv.calldir) + offset; 1255b76ce561STrond Myklebust used = xdr_skb_read_bits(desc, p, len); 1256d1109aa5STrond Myklebust transport->recv.offset += used; 125718dca02aSRicardo Labiaga if (used != len) 125818dca02aSRicardo Labiaga return; 1259d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_READ_CALLDIR; 1260f4a2e418SRicardo Labiaga /* 1261f4a2e418SRicardo Labiaga * We don't yet have the XDR buffer, so we will write the calldir 1262f4a2e418SRicardo Labiaga * out after we get the buffer from the 'struct rpc_rqst' 1263f4a2e418SRicardo Labiaga */ 1264d1109aa5STrond Myklebust switch (ntohl(transport->recv.calldir)) { 1265b76ce561STrond Myklebust case RPC_REPLY: 1266d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_CALLDIR; 1267d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_DATA; 1268d1109aa5STrond Myklebust transport->recv.flags |= TCP_RPC_REPLY; 1269b76ce561STrond Myklebust break; 1270b76ce561STrond Myklebust case RPC_CALL: 1271d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_CALLDIR; 1272d1109aa5STrond Myklebust transport->recv.flags |= TCP_RCV_COPY_DATA; 1273d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RPC_REPLY; 1274b76ce561STrond Myklebust break; 1275b76ce561STrond Myklebust default: 1276b76ce561STrond Myklebust dprintk("RPC: invalid request message type\n"); 1277a519fc7aSTrond Myklebust xs_tcp_force_close(&transport->xprt); 1278b76ce561STrond Myklebust } 127918dca02aSRicardo Labiaga xs_tcp_check_fraghdr(transport); 128018dca02aSRicardo Labiaga } 128118dca02aSRicardo Labiaga 128244b98efdSRicardo Labiaga static inline void xs_tcp_read_common(struct rpc_xprt *xprt, 128344b98efdSRicardo Labiaga struct xdr_skb_reader *desc, 128444b98efdSRicardo Labiaga struct rpc_rqst *req) 1285a246b010SChuck Lever { 128644b98efdSRicardo Labiaga struct sock_xprt *transport = 128744b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 1288a246b010SChuck Lever struct xdr_buf *rcvbuf; 1289a246b010SChuck Lever size_t len; 1290a246b010SChuck Lever ssize_t r; 1291a246b010SChuck Lever 1292a246b010SChuck Lever rcvbuf = &req->rq_private_buf; 1293f4a2e418SRicardo Labiaga 1294d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_COPY_CALLDIR) { 1295f4a2e418SRicardo Labiaga /* 1296f4a2e418SRicardo Labiaga * Save the RPC direction in the XDR buffer 1297f4a2e418SRicardo Labiaga */ 1298d1109aa5STrond Myklebust memcpy(rcvbuf->head[0].iov_base + transport->recv.copied, 1299d1109aa5STrond Myklebust &transport->recv.calldir, 1300d1109aa5STrond Myklebust sizeof(transport->recv.calldir)); 1301d1109aa5STrond Myklebust transport->recv.copied += sizeof(transport->recv.calldir); 1302d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_CALLDIR; 1303a246b010SChuck Lever } 1304a246b010SChuck Lever 1305a246b010SChuck Lever len = desc->count; 1306d1109aa5STrond Myklebust if (len > transport->recv.len - transport->recv.offset) 1307d1109aa5STrond Myklebust desc->count = transport->recv.len - transport->recv.offset; 1308d1109aa5STrond Myklebust r = xdr_partial_copy_from_skb(rcvbuf, transport->recv.copied, 13099d292316SChuck Lever desc, xdr_skb_read_bits); 1310a246b010SChuck Lever 1311040249dfSTrond Myklebust if (desc->count) { 1312a246b010SChuck Lever /* Error when copying to the receive buffer, 1313a246b010SChuck Lever * usually because we weren't able to allocate 1314a246b010SChuck Lever * additional buffer pages. All we can do now 1315e136d092SChuck Lever * is turn off TCP_RCV_COPY_DATA, so the request 1316a246b010SChuck Lever * will not receive any additional updates, 1317a246b010SChuck Lever * and time out. 1318a246b010SChuck Lever * Any remaining data from this record will 1319a246b010SChuck Lever * be discarded. 1320a246b010SChuck Lever */ 1321d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_DATA; 1322a246b010SChuck Lever dprintk("RPC: XID %08x truncated request\n", 1323d1109aa5STrond Myklebust ntohl(transport->recv.xid)); 1324d1109aa5STrond Myklebust dprintk("RPC: xprt = %p, recv.copied = %lu, " 1325d1109aa5STrond Myklebust "recv.offset = %u, recv.len = %u\n", 1326d1109aa5STrond Myklebust xprt, transport->recv.copied, 1327d1109aa5STrond Myklebust transport->recv.offset, transport->recv.len); 132844b98efdSRicardo Labiaga return; 1329a246b010SChuck Lever } 1330a246b010SChuck Lever 1331d1109aa5STrond Myklebust transport->recv.copied += r; 1332d1109aa5STrond Myklebust transport->recv.offset += r; 1333040249dfSTrond Myklebust desc->count = len - r; 1334040249dfSTrond Myklebust 13355b5e0928SAlexey Dobriyan dprintk("RPC: XID %08x read %zd bytes\n", 1336d1109aa5STrond Myklebust ntohl(transport->recv.xid), r); 1337d1109aa5STrond Myklebust dprintk("RPC: xprt = %p, recv.copied = %lu, recv.offset = %u, " 1338d1109aa5STrond Myklebust "recv.len = %u\n", xprt, transport->recv.copied, 1339d1109aa5STrond Myklebust transport->recv.offset, transport->recv.len); 1340a246b010SChuck Lever 1341d1109aa5STrond Myklebust if (transport->recv.copied == req->rq_private_buf.buflen) 1342d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_DATA; 1343d1109aa5STrond Myklebust else if (transport->recv.offset == transport->recv.len) { 1344d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_LAST_FRAG) 1345d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_DATA; 1346a246b010SChuck Lever } 134744b98efdSRicardo Labiaga } 134844b98efdSRicardo Labiaga 134944b98efdSRicardo Labiaga /* 135044b98efdSRicardo Labiaga * Finds the request corresponding to the RPC xid and invokes the common 135144b98efdSRicardo Labiaga * tcp read code to read the data. 135244b98efdSRicardo Labiaga */ 135344b98efdSRicardo Labiaga static inline int xs_tcp_read_reply(struct rpc_xprt *xprt, 135444b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 135544b98efdSRicardo Labiaga { 135644b98efdSRicardo Labiaga struct sock_xprt *transport = 135744b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 135844b98efdSRicardo Labiaga struct rpc_rqst *req; 135944b98efdSRicardo Labiaga 1360d1109aa5STrond Myklebust dprintk("RPC: read reply XID %08x\n", ntohl(transport->recv.xid)); 136144b98efdSRicardo Labiaga 136244b98efdSRicardo Labiaga /* Find and lock the request corresponding to this xid */ 136375c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1364d1109aa5STrond Myklebust req = xprt_lookup_rqst(xprt, transport->recv.xid); 136544b98efdSRicardo Labiaga if (!req) { 136644b98efdSRicardo Labiaga dprintk("RPC: XID %08x request not found!\n", 1367d1109aa5STrond Myklebust ntohl(transport->recv.xid)); 136875c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 136944b98efdSRicardo Labiaga return -1; 137044b98efdSRicardo Labiaga } 1371729749bbSTrond Myklebust xprt_pin_rqst(req); 137275c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 137344b98efdSRicardo Labiaga 137444b98efdSRicardo Labiaga xs_tcp_read_common(xprt, desc, req); 137544b98efdSRicardo Labiaga 137675c84151STrond Myklebust spin_lock(&xprt->queue_lock); 1377d1109aa5STrond Myklebust if (!(transport->recv.flags & TCP_RCV_COPY_DATA)) 1378d1109aa5STrond Myklebust xprt_complete_rqst(req->rq_task, transport->recv.copied); 1379729749bbSTrond Myklebust xprt_unpin_rqst(req); 138075c84151STrond Myklebust spin_unlock(&xprt->queue_lock); 138144b98efdSRicardo Labiaga return 0; 138244b98efdSRicardo Labiaga } 138344b98efdSRicardo Labiaga 13849e00abc3STrond Myklebust #if defined(CONFIG_SUNRPC_BACKCHANNEL) 138544b98efdSRicardo Labiaga /* 138644b98efdSRicardo Labiaga * Obtains an rpc_rqst previously allocated and invokes the common 138744b98efdSRicardo Labiaga * tcp read code to read the data. The result is placed in the callback 138844b98efdSRicardo Labiaga * queue. 138944b98efdSRicardo Labiaga * If we're unable to obtain the rpc_rqst we schedule the closing of the 139044b98efdSRicardo Labiaga * connection and return -1. 139144b98efdSRicardo Labiaga */ 13922ea24497STrond Myklebust static int xs_tcp_read_callback(struct rpc_xprt *xprt, 139344b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 139444b98efdSRicardo Labiaga { 139544b98efdSRicardo Labiaga struct sock_xprt *transport = 139644b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 139744b98efdSRicardo Labiaga struct rpc_rqst *req; 139844b98efdSRicardo Labiaga 1399c89091c8STrond Myklebust /* Look up the request corresponding to the given XID */ 1400d1109aa5STrond Myklebust req = xprt_lookup_bc_request(xprt, transport->recv.xid); 140144b98efdSRicardo Labiaga if (req == NULL) { 140244b98efdSRicardo Labiaga printk(KERN_WARNING "Callback slot table overflowed\n"); 140344b98efdSRicardo Labiaga xprt_force_disconnect(xprt); 140444b98efdSRicardo Labiaga return -1; 140544b98efdSRicardo Labiaga } 140644b98efdSRicardo Labiaga 140744b98efdSRicardo Labiaga dprintk("RPC: read callback XID %08x\n", ntohl(req->rq_xid)); 140844b98efdSRicardo Labiaga xs_tcp_read_common(xprt, desc, req); 140944b98efdSRicardo Labiaga 1410d1109aa5STrond Myklebust if (!(transport->recv.flags & TCP_RCV_COPY_DATA)) 1411d1109aa5STrond Myklebust xprt_complete_bc_request(req, transport->recv.copied); 141244b98efdSRicardo Labiaga 141344b98efdSRicardo Labiaga return 0; 141444b98efdSRicardo Labiaga } 141544b98efdSRicardo Labiaga 141644b98efdSRicardo Labiaga static inline int _xs_tcp_read_data(struct rpc_xprt *xprt, 141744b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 141844b98efdSRicardo Labiaga { 141944b98efdSRicardo Labiaga struct sock_xprt *transport = 142044b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 142144b98efdSRicardo Labiaga 1422d1109aa5STrond Myklebust return (transport->recv.flags & TCP_RPC_REPLY) ? 142344b98efdSRicardo Labiaga xs_tcp_read_reply(xprt, desc) : 142444b98efdSRicardo Labiaga xs_tcp_read_callback(xprt, desc); 142544b98efdSRicardo Labiaga } 142676566773SChuck Lever 142776566773SChuck Lever static int xs_tcp_bc_up(struct svc_serv *serv, struct net *net) 142876566773SChuck Lever { 142976566773SChuck Lever int ret; 143076566773SChuck Lever 143176566773SChuck Lever ret = svc_create_xprt(serv, "tcp-bc", net, PF_INET, 0, 143276566773SChuck Lever SVC_SOCK_ANONYMOUS); 143376566773SChuck Lever if (ret < 0) 143476566773SChuck Lever return ret; 143576566773SChuck Lever return 0; 143676566773SChuck Lever } 14376b26cc8cSChuck Lever 14386b26cc8cSChuck Lever static size_t xs_tcp_bc_maxpayload(struct rpc_xprt *xprt) 14396b26cc8cSChuck Lever { 14406b26cc8cSChuck Lever return PAGE_SIZE; 14416b26cc8cSChuck Lever } 144244b98efdSRicardo Labiaga #else 144344b98efdSRicardo Labiaga static inline int _xs_tcp_read_data(struct rpc_xprt *xprt, 144444b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 144544b98efdSRicardo Labiaga { 144644b98efdSRicardo Labiaga return xs_tcp_read_reply(xprt, desc); 144744b98efdSRicardo Labiaga } 14489e00abc3STrond Myklebust #endif /* CONFIG_SUNRPC_BACKCHANNEL */ 144944b98efdSRicardo Labiaga 145044b98efdSRicardo Labiaga /* 145144b98efdSRicardo Labiaga * Read data off the transport. This can be either an RPC_CALL or an 145244b98efdSRicardo Labiaga * RPC_REPLY. Relay the processing to helper functions. 145344b98efdSRicardo Labiaga */ 145444b98efdSRicardo Labiaga static void xs_tcp_read_data(struct rpc_xprt *xprt, 145544b98efdSRicardo Labiaga struct xdr_skb_reader *desc) 145644b98efdSRicardo Labiaga { 145744b98efdSRicardo Labiaga struct sock_xprt *transport = 145844b98efdSRicardo Labiaga container_of(xprt, struct sock_xprt, xprt); 145944b98efdSRicardo Labiaga 146044b98efdSRicardo Labiaga if (_xs_tcp_read_data(xprt, desc) == 0) 146151971139SChuck Lever xs_tcp_check_fraghdr(transport); 146244b98efdSRicardo Labiaga else { 146344b98efdSRicardo Labiaga /* 146444b98efdSRicardo Labiaga * The transport_lock protects the request handling. 1465d1109aa5STrond Myklebust * There's no need to hold it to update the recv.flags. 146644b98efdSRicardo Labiaga */ 1467d1109aa5STrond Myklebust transport->recv.flags &= ~TCP_RCV_COPY_DATA; 146844b98efdSRicardo Labiaga } 1469a246b010SChuck Lever } 1470a246b010SChuck Lever 1471dd456471SChuck Lever static inline void xs_tcp_read_discard(struct sock_xprt *transport, struct xdr_skb_reader *desc) 1472a246b010SChuck Lever { 1473a246b010SChuck Lever size_t len; 1474a246b010SChuck Lever 1475d1109aa5STrond Myklebust len = transport->recv.len - transport->recv.offset; 1476a246b010SChuck Lever if (len > desc->count) 1477a246b010SChuck Lever len = desc->count; 1478a246b010SChuck Lever desc->count -= len; 1479a246b010SChuck Lever desc->offset += len; 1480d1109aa5STrond Myklebust transport->recv.offset += len; 14815b5e0928SAlexey Dobriyan dprintk("RPC: discarded %zu bytes\n", len); 148251971139SChuck Lever xs_tcp_check_fraghdr(transport); 1483a246b010SChuck Lever } 1484a246b010SChuck Lever 14859903cd1cSChuck Lever static int xs_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, unsigned int offset, size_t len) 1486a246b010SChuck Lever { 1487a246b010SChuck Lever struct rpc_xprt *xprt = rd_desc->arg.data; 148851971139SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1489dd456471SChuck Lever struct xdr_skb_reader desc = { 1490a246b010SChuck Lever .skb = skb, 1491a246b010SChuck Lever .offset = offset, 1492a246b010SChuck Lever .count = len, 1493a246b010SChuck Lever }; 14943d188805STrond Myklebust size_t ret; 1495a246b010SChuck Lever 14969903cd1cSChuck Lever dprintk("RPC: xs_tcp_data_recv started\n"); 1497a246b010SChuck Lever do { 14981a867a08SJeff Layton trace_xs_tcp_data_recv(transport); 1499a246b010SChuck Lever /* Read in a new fragment marker if necessary */ 1500a246b010SChuck Lever /* Can we ever really expect to get completely empty fragments? */ 1501d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_COPY_FRAGHDR) { 15029903cd1cSChuck Lever xs_tcp_read_fraghdr(xprt, &desc); 1503a246b010SChuck Lever continue; 1504a246b010SChuck Lever } 1505a246b010SChuck Lever /* Read in the xid if necessary */ 1506d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_COPY_XID) { 150751971139SChuck Lever xs_tcp_read_xid(transport, &desc); 1508a246b010SChuck Lever continue; 1509a246b010SChuck Lever } 151018dca02aSRicardo Labiaga /* Read in the call/reply flag */ 1511d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_READ_CALLDIR) { 151218dca02aSRicardo Labiaga xs_tcp_read_calldir(transport, &desc); 151318dca02aSRicardo Labiaga continue; 151418dca02aSRicardo Labiaga } 1515a246b010SChuck Lever /* Read in the request data */ 1516d1109aa5STrond Myklebust if (transport->recv.flags & TCP_RCV_COPY_DATA) { 151744b98efdSRicardo Labiaga xs_tcp_read_data(xprt, &desc); 1518a246b010SChuck Lever continue; 1519a246b010SChuck Lever } 1520a246b010SChuck Lever /* Skip over any trailing bytes on short reads */ 152151971139SChuck Lever xs_tcp_read_discard(transport, &desc); 1522a246b010SChuck Lever } while (desc.count); 15233d188805STrond Myklebust ret = len - desc.count; 15243d188805STrond Myklebust if (ret < rd_desc->count) 15253d188805STrond Myklebust rd_desc->count -= ret; 15263d188805STrond Myklebust else 15273d188805STrond Myklebust rd_desc->count = 0; 15281a867a08SJeff Layton trace_xs_tcp_data_recv(transport); 15299903cd1cSChuck Lever dprintk("RPC: xs_tcp_data_recv done\n"); 15303d188805STrond Myklebust return ret; 1531a246b010SChuck Lever } 1532a246b010SChuck Lever 153366d7a56aSTrond Myklebust static void xs_tcp_data_receive(struct sock_xprt *transport) 153466d7a56aSTrond Myklebust { 153566d7a56aSTrond Myklebust struct rpc_xprt *xprt = &transport->xprt; 153666d7a56aSTrond Myklebust struct sock *sk; 153766d7a56aSTrond Myklebust read_descriptor_t rd_desc = { 153866d7a56aSTrond Myklebust .arg.data = xprt, 153966d7a56aSTrond Myklebust }; 154066d7a56aSTrond Myklebust unsigned long total = 0; 154166d7a56aSTrond Myklebust int read = 0; 154266d7a56aSTrond Myklebust 15430af3442aSTrond Myklebust restart: 1544edc1b01cSTrond Myklebust mutex_lock(&transport->recv_mutex); 154566d7a56aSTrond Myklebust sk = transport->inet; 1546edc1b01cSTrond Myklebust if (sk == NULL) 1547edc1b01cSTrond Myklebust goto out; 154866d7a56aSTrond Myklebust 154966d7a56aSTrond Myklebust /* We use rd_desc to pass struct xprt to xs_tcp_data_recv */ 15500af3442aSTrond Myklebust for (;;) { 15513d188805STrond Myklebust rd_desc.count = RPC_TCP_READ_CHUNK_SZ; 1552edc1b01cSTrond Myklebust lock_sock(sk); 155366d7a56aSTrond Myklebust read = tcp_read_sock(sk, &rd_desc, xs_tcp_data_recv); 15543d188805STrond Myklebust if (rd_desc.count != 0 || read < 0) { 155542d42a5bSTrond Myklebust clear_bit(XPRT_SOCK_DATA_READY, &transport->sock_state); 1556edc1b01cSTrond Myklebust release_sock(sk); 155766d7a56aSTrond Myklebust break; 15588d6f97d6STrond Myklebust } 155942d42a5bSTrond Myklebust release_sock(sk); 156066d7a56aSTrond Myklebust total += read; 15610af3442aSTrond Myklebust if (need_resched()) { 15620af3442aSTrond Myklebust mutex_unlock(&transport->recv_mutex); 15630af3442aSTrond Myklebust cond_resched(); 15640af3442aSTrond Myklebust goto restart; 15650af3442aSTrond Myklebust } 156666d7a56aSTrond Myklebust } 15678d6f97d6STrond Myklebust if (test_bit(XPRT_SOCK_DATA_READY, &transport->sock_state)) 15688d6f97d6STrond Myklebust queue_work(xprtiod_workqueue, &transport->recv_worker); 1569edc1b01cSTrond Myklebust out: 1570edc1b01cSTrond Myklebust mutex_unlock(&transport->recv_mutex); 157166d7a56aSTrond Myklebust trace_xs_tcp_data_ready(xprt, read, total); 157266d7a56aSTrond Myklebust } 157366d7a56aSTrond Myklebust 1574edc1b01cSTrond Myklebust static void xs_tcp_data_receive_workfn(struct work_struct *work) 1575edc1b01cSTrond Myklebust { 1576edc1b01cSTrond Myklebust struct sock_xprt *transport = 1577edc1b01cSTrond Myklebust container_of(work, struct sock_xprt, recv_worker); 1578edc1b01cSTrond Myklebust xs_tcp_data_receive(transport); 1579edc1b01cSTrond Myklebust } 1580edc1b01cSTrond Myklebust 15819903cd1cSChuck Lever /** 15829903cd1cSChuck Lever * xs_tcp_state_change - callback to handle TCP socket state changes 15839903cd1cSChuck Lever * @sk: socket whose state has changed 15849903cd1cSChuck Lever * 15859903cd1cSChuck Lever */ 15869903cd1cSChuck Lever static void xs_tcp_state_change(struct sock *sk) 1587a246b010SChuck Lever { 1588a246b010SChuck Lever struct rpc_xprt *xprt; 15890fdea1e8STrond Myklebust struct sock_xprt *transport; 1590a246b010SChuck Lever 1591f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1592a246b010SChuck Lever if (!(xprt = xprt_from_sock(sk))) 1593a246b010SChuck Lever goto out; 15949903cd1cSChuck Lever dprintk("RPC: xs_tcp_state_change client %p...\n", xprt); 1595669502ffSAndy Chittenden dprintk("RPC: state %x conn %d dead %d zapped %d sk_shutdown %d\n", 1596a246b010SChuck Lever sk->sk_state, xprt_connected(xprt), 1597a246b010SChuck Lever sock_flag(sk, SOCK_DEAD), 1598669502ffSAndy Chittenden sock_flag(sk, SOCK_ZAPPED), 1599669502ffSAndy Chittenden sk->sk_shutdown); 1600a246b010SChuck Lever 16010fdea1e8STrond Myklebust transport = container_of(xprt, struct sock_xprt, xprt); 160240b5ea0cSTrond Myklebust trace_rpc_socket_state_change(xprt, sk->sk_socket); 1603a246b010SChuck Lever switch (sk->sk_state) { 1604a246b010SChuck Lever case TCP_ESTABLISHED: 1605f064af1eSEric Dumazet spin_lock(&xprt->transport_lock); 1606a246b010SChuck Lever if (!xprt_test_and_set_connected(xprt)) { 16078b71798cSTrond Myklebust xprt->connect_cookie++; 16080fdea1e8STrond Myklebust clear_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 16090fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 161051971139SChuck Lever 16112a491991STrond Myklebust xprt_wake_pending_tasks(xprt, -EAGAIN); 1612a246b010SChuck Lever } 1613f064af1eSEric Dumazet spin_unlock(&xprt->transport_lock); 1614a246b010SChuck Lever break; 16153b948ae5STrond Myklebust case TCP_FIN_WAIT1: 16163b948ae5STrond Myklebust /* The client initiated a shutdown of the socket */ 16177c1d71cfSTrond Myklebust xprt->connect_cookie++; 1618663b8858STrond Myklebust xprt->reestablish_timeout = 0; 16193b948ae5STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 16204e857c58SPeter Zijlstra smp_mb__before_atomic(); 16213b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1622ef803670STrond Myklebust clear_bit(XPRT_CLOSE_WAIT, &xprt->state); 16234e857c58SPeter Zijlstra smp_mb__after_atomic(); 1624a246b010SChuck Lever break; 1625632e3bdcSTrond Myklebust case TCP_CLOSE_WAIT: 16263b948ae5STrond Myklebust /* The server initiated a shutdown of the socket */ 16277c1d71cfSTrond Myklebust xprt->connect_cookie++; 1628d0bea455STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 1629a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 1630e9d47639SGustavo A. R. Silva /* fall through */ 1631663b8858STrond Myklebust case TCP_CLOSING: 1632663b8858STrond Myklebust /* 1633663b8858STrond Myklebust * If the server closed down the connection, make sure that 1634663b8858STrond Myklebust * we back off before reconnecting 1635663b8858STrond Myklebust */ 1636663b8858STrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 1637663b8858STrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 16383b948ae5STrond Myklebust break; 16393b948ae5STrond Myklebust case TCP_LAST_ACK: 1640670f9457STrond Myklebust set_bit(XPRT_CLOSING, &xprt->state); 16414e857c58SPeter Zijlstra smp_mb__before_atomic(); 16423b948ae5STrond Myklebust clear_bit(XPRT_CONNECTED, &xprt->state); 16434e857c58SPeter Zijlstra smp_mb__after_atomic(); 16443b948ae5STrond Myklebust break; 16453b948ae5STrond Myklebust case TCP_CLOSE: 16460fdea1e8STrond Myklebust if (test_and_clear_bit(XPRT_SOCK_CONNECTING, 16470fdea1e8STrond Myklebust &transport->sock_state)) 16480fdea1e8STrond Myklebust xprt_clear_connecting(xprt); 16499b30889cSTrond Myklebust clear_bit(XPRT_CLOSING, &xprt->state); 16503ffbc1d6SNeilBrown if (sk->sk_err) 16513ffbc1d6SNeilBrown xprt_wake_pending_tasks(xprt, -sk->sk_err); 16529b30889cSTrond Myklebust /* Trigger the socket release */ 16539b30889cSTrond Myklebust xs_tcp_force_close(xprt); 1654a246b010SChuck Lever } 1655a246b010SChuck Lever out: 1656f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1657a246b010SChuck Lever } 1658a246b010SChuck Lever 16591f0fa154SIlpo Järvinen static void xs_write_space(struct sock *sk) 16601f0fa154SIlpo Järvinen { 166113331a55STrond Myklebust struct socket_wq *wq; 16621f0fa154SIlpo Järvinen struct rpc_xprt *xprt; 16631f0fa154SIlpo Järvinen 166413331a55STrond Myklebust if (!sk->sk_socket) 16651f0fa154SIlpo Järvinen return; 166613331a55STrond Myklebust clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 16671f0fa154SIlpo Järvinen 16681f0fa154SIlpo Järvinen if (unlikely(!(xprt = xprt_from_sock(sk)))) 16691f0fa154SIlpo Järvinen return; 167013331a55STrond Myklebust rcu_read_lock(); 167113331a55STrond Myklebust wq = rcu_dereference(sk->sk_wq); 167213331a55STrond Myklebust if (!wq || test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &wq->flags) == 0) 167313331a55STrond Myklebust goto out; 16741f0fa154SIlpo Järvinen 16751f0fa154SIlpo Järvinen xprt_write_space(xprt); 167613331a55STrond Myklebust out: 167713331a55STrond Myklebust rcu_read_unlock(); 16781f0fa154SIlpo Järvinen } 16791f0fa154SIlpo Järvinen 16802a9e1cfaSTrond Myklebust /** 1681c7b2cae8SChuck Lever * xs_udp_write_space - callback invoked when socket buffer space 1682c7b2cae8SChuck Lever * becomes available 16839903cd1cSChuck Lever * @sk: socket whose state has changed 16849903cd1cSChuck Lever * 1685a246b010SChuck Lever * Called when more output buffer space is available for this socket. 1686a246b010SChuck Lever * We try not to wake our writers until they can make "significant" 1687c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1688a246b010SChuck Lever * with a bunch of small requests. 1689a246b010SChuck Lever */ 1690c7b2cae8SChuck Lever static void xs_udp_write_space(struct sock *sk) 1691a246b010SChuck Lever { 1692f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1693c7b2cae8SChuck Lever 1694c7b2cae8SChuck Lever /* from net/core/sock.c:sock_def_write_space */ 16951f0fa154SIlpo Järvinen if (sock_writeable(sk)) 16961f0fa154SIlpo Järvinen xs_write_space(sk); 1697c7b2cae8SChuck Lever 1698f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1699c7b2cae8SChuck Lever } 1700c7b2cae8SChuck Lever 1701c7b2cae8SChuck Lever /** 1702c7b2cae8SChuck Lever * xs_tcp_write_space - callback invoked when socket buffer space 1703c7b2cae8SChuck Lever * becomes available 1704c7b2cae8SChuck Lever * @sk: socket whose state has changed 1705c7b2cae8SChuck Lever * 1706c7b2cae8SChuck Lever * Called when more output buffer space is available for this socket. 1707c7b2cae8SChuck Lever * We try not to wake our writers until they can make "significant" 1708c7b2cae8SChuck Lever * progress, otherwise we'll waste resources thrashing kernel_sendmsg 1709c7b2cae8SChuck Lever * with a bunch of small requests. 1710c7b2cae8SChuck Lever */ 1711c7b2cae8SChuck Lever static void xs_tcp_write_space(struct sock *sk) 1712c7b2cae8SChuck Lever { 1713f064af1eSEric Dumazet read_lock_bh(&sk->sk_callback_lock); 1714c7b2cae8SChuck Lever 1715c7b2cae8SChuck Lever /* from net/core/stream.c:sk_stream_write_space */ 171664dc6130SEric Dumazet if (sk_stream_is_writeable(sk)) 17171f0fa154SIlpo Järvinen xs_write_space(sk); 1718c7b2cae8SChuck Lever 1719f064af1eSEric Dumazet read_unlock_bh(&sk->sk_callback_lock); 1720a246b010SChuck Lever } 1721a246b010SChuck Lever 1722470056c2SChuck Lever static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) 1723a246b010SChuck Lever { 1724ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 1725ee0ac0c2SChuck Lever struct sock *sk = transport->inet; 1726a246b010SChuck Lever 17277c6e066eSChuck Lever if (transport->rcvsize) { 1728a246b010SChuck Lever sk->sk_userlocks |= SOCK_RCVBUF_LOCK; 17297c6e066eSChuck Lever sk->sk_rcvbuf = transport->rcvsize * xprt->max_reqs * 2; 1730a246b010SChuck Lever } 17317c6e066eSChuck Lever if (transport->sndsize) { 1732a246b010SChuck Lever sk->sk_userlocks |= SOCK_SNDBUF_LOCK; 17337c6e066eSChuck Lever sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; 1734a246b010SChuck Lever sk->sk_write_space(sk); 1735a246b010SChuck Lever } 1736a246b010SChuck Lever } 1737a246b010SChuck Lever 173843118c29SChuck Lever /** 1739470056c2SChuck Lever * xs_udp_set_buffer_size - set send and receive limits 174043118c29SChuck Lever * @xprt: generic transport 1741470056c2SChuck Lever * @sndsize: requested size of send buffer, in bytes 1742470056c2SChuck Lever * @rcvsize: requested size of receive buffer, in bytes 174343118c29SChuck Lever * 1744470056c2SChuck Lever * Set socket send and receive buffer size limits. 174543118c29SChuck Lever */ 1746470056c2SChuck Lever static void xs_udp_set_buffer_size(struct rpc_xprt *xprt, size_t sndsize, size_t rcvsize) 174743118c29SChuck Lever { 17487c6e066eSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 17497c6e066eSChuck Lever 17507c6e066eSChuck Lever transport->sndsize = 0; 1751470056c2SChuck Lever if (sndsize) 17527c6e066eSChuck Lever transport->sndsize = sndsize + 1024; 17537c6e066eSChuck Lever transport->rcvsize = 0; 1754470056c2SChuck Lever if (rcvsize) 17557c6e066eSChuck Lever transport->rcvsize = rcvsize + 1024; 1756470056c2SChuck Lever 1757470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 175843118c29SChuck Lever } 175943118c29SChuck Lever 176046c0ee8bSChuck Lever /** 176146c0ee8bSChuck Lever * xs_udp_timer - called when a retransmit timeout occurs on a UDP transport 176246c0ee8bSChuck Lever * @task: task that timed out 176346c0ee8bSChuck Lever * 176446c0ee8bSChuck Lever * Adjust the congestion window after a retransmit timeout has occurred. 176546c0ee8bSChuck Lever */ 17666a24dfb6STrond Myklebust static void xs_udp_timer(struct rpc_xprt *xprt, struct rpc_task *task) 176746c0ee8bSChuck Lever { 1768b977b644SChuck Lever spin_lock_bh(&xprt->transport_lock); 17696a24dfb6STrond Myklebust xprt_adjust_cwnd(xprt, task, -ETIMEDOUT); 1770b977b644SChuck Lever spin_unlock_bh(&xprt->transport_lock); 177146c0ee8bSChuck Lever } 177246c0ee8bSChuck Lever 1773b85d8806SChuck Lever static unsigned short xs_get_random_port(void) 1774b85d8806SChuck Lever { 17755d71899aSFrank Sorenson unsigned short range = xprt_max_resvport - xprt_min_resvport + 1; 177663862b5bSAruna-Hewapathirane unsigned short rand = (unsigned short) prandom_u32() % range; 1777b85d8806SChuck Lever return rand + xprt_min_resvport; 1778b85d8806SChuck Lever } 1779b85d8806SChuck Lever 178092200412SChuck Lever /** 17814dda9c8aSTrond Myklebust * xs_set_reuseaddr_port - set the socket's port and address reuse options 17824dda9c8aSTrond Myklebust * @sock: socket 17834dda9c8aSTrond Myklebust * 17844dda9c8aSTrond Myklebust * Note that this function has to be called on all sockets that share the 17854dda9c8aSTrond Myklebust * same port, and it must be called before binding. 17864dda9c8aSTrond Myklebust */ 17874dda9c8aSTrond Myklebust static void xs_sock_set_reuseport(struct socket *sock) 17884dda9c8aSTrond Myklebust { 1789402e23b4STrond Myklebust int opt = 1; 17904dda9c8aSTrond Myklebust 1791402e23b4STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_REUSEPORT, 1792402e23b4STrond Myklebust (char *)&opt, sizeof(opt)); 17934dda9c8aSTrond Myklebust } 17944dda9c8aSTrond Myklebust 17954dda9c8aSTrond Myklebust static unsigned short xs_sock_getport(struct socket *sock) 17964dda9c8aSTrond Myklebust { 17974dda9c8aSTrond Myklebust struct sockaddr_storage buf; 17984dda9c8aSTrond Myklebust unsigned short port = 0; 17994dda9c8aSTrond Myklebust 18009b2c45d4SDenys Vlasenko if (kernel_getsockname(sock, (struct sockaddr *)&buf) < 0) 18014dda9c8aSTrond Myklebust goto out; 18024dda9c8aSTrond Myklebust switch (buf.ss_family) { 18034dda9c8aSTrond Myklebust case AF_INET6: 18044dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in6 *)&buf)->sin6_port); 18054dda9c8aSTrond Myklebust break; 18064dda9c8aSTrond Myklebust case AF_INET: 18074dda9c8aSTrond Myklebust port = ntohs(((struct sockaddr_in *)&buf)->sin_port); 18084dda9c8aSTrond Myklebust } 18094dda9c8aSTrond Myklebust out: 18104dda9c8aSTrond Myklebust return port; 18114dda9c8aSTrond Myklebust } 18124dda9c8aSTrond Myklebust 18134dda9c8aSTrond Myklebust /** 181492200412SChuck Lever * xs_set_port - reset the port number in the remote endpoint address 181592200412SChuck Lever * @xprt: generic transport 181692200412SChuck Lever * @port: new port number 181792200412SChuck Lever * 181892200412SChuck Lever */ 181992200412SChuck Lever static void xs_set_port(struct rpc_xprt *xprt, unsigned short port) 182092200412SChuck Lever { 182192200412SChuck Lever dprintk("RPC: setting port for xprt %p to %u\n", xprt, port); 1822c4efcb1dSChuck Lever 18239dc3b095SChuck Lever rpc_set_port(xs_addr(xprt), port); 18249dc3b095SChuck Lever xs_update_peer_port(xprt); 182592200412SChuck Lever } 182692200412SChuck Lever 18274dda9c8aSTrond Myklebust static void xs_set_srcport(struct sock_xprt *transport, struct socket *sock) 18284dda9c8aSTrond Myklebust { 18294dda9c8aSTrond Myklebust if (transport->srcport == 0) 18304dda9c8aSTrond Myklebust transport->srcport = xs_sock_getport(sock); 18314dda9c8aSTrond Myklebust } 18324dda9c8aSTrond Myklebust 18335d4ec932SPavel Emelyanov static unsigned short xs_get_srcport(struct sock_xprt *transport) 183467a391d7STrond Myklebust { 1835fbfffbd5SChuck Lever unsigned short port = transport->srcport; 183667a391d7STrond Myklebust 183767a391d7STrond Myklebust if (port == 0 && transport->xprt.resvport) 183867a391d7STrond Myklebust port = xs_get_random_port(); 183967a391d7STrond Myklebust return port; 184067a391d7STrond Myklebust } 184167a391d7STrond Myklebust 1842baaf4e48SPavel Emelyanov static unsigned short xs_next_srcport(struct sock_xprt *transport, unsigned short port) 184367a391d7STrond Myklebust { 1844fbfffbd5SChuck Lever if (transport->srcport != 0) 1845fbfffbd5SChuck Lever transport->srcport = 0; 184667a391d7STrond Myklebust if (!transport->xprt.resvport) 184767a391d7STrond Myklebust return 0; 184867a391d7STrond Myklebust if (port <= xprt_min_resvport || port > xprt_max_resvport) 184967a391d7STrond Myklebust return xprt_max_resvport; 185067a391d7STrond Myklebust return --port; 185167a391d7STrond Myklebust } 1852beb59b68SPavel Emelyanov static int xs_bind(struct sock_xprt *transport, struct socket *sock) 1853a246b010SChuck Lever { 1854beb59b68SPavel Emelyanov struct sockaddr_storage myaddr; 185567a391d7STrond Myklebust int err, nloop = 0; 18565d4ec932SPavel Emelyanov unsigned short port = xs_get_srcport(transport); 185767a391d7STrond Myklebust unsigned short last; 1858a246b010SChuck Lever 18590f7a622cSChris Perl /* 18600f7a622cSChris Perl * If we are asking for any ephemeral port (i.e. port == 0 && 18610f7a622cSChris Perl * transport->xprt.resvport == 0), don't bind. Let the local 18620f7a622cSChris Perl * port selection happen implicitly when the socket is used 18630f7a622cSChris Perl * (for example at connect time). 18640f7a622cSChris Perl * 18650f7a622cSChris Perl * This ensures that we can continue to establish TCP 18660f7a622cSChris Perl * connections even when all local ephemeral ports are already 18670f7a622cSChris Perl * a part of some TCP connection. This makes no difference 18680f7a622cSChris Perl * for UDP sockets, but also doens't harm them. 18690f7a622cSChris Perl * 18700f7a622cSChris Perl * If we're asking for any reserved port (i.e. port == 0 && 18710f7a622cSChris Perl * transport->xprt.resvport == 1) xs_get_srcport above will 18720f7a622cSChris Perl * ensure that port is non-zero and we will bind as needed. 18730f7a622cSChris Perl */ 18740f7a622cSChris Perl if (port == 0) 18750f7a622cSChris Perl return 0; 18760f7a622cSChris Perl 1877beb59b68SPavel Emelyanov memcpy(&myaddr, &transport->srcaddr, transport->xprt.addrlen); 1878a246b010SChuck Lever do { 1879beb59b68SPavel Emelyanov rpc_set_port((struct sockaddr *)&myaddr, port); 1880e6242e92SSridhar Samudrala err = kernel_bind(sock, (struct sockaddr *)&myaddr, 1881beb59b68SPavel Emelyanov transport->xprt.addrlen); 1882a246b010SChuck Lever if (err == 0) { 1883fbfffbd5SChuck Lever transport->srcport = port; 1884d3bc9a1dSFrank van Maarseveen break; 1885a246b010SChuck Lever } 188667a391d7STrond Myklebust last = port; 1887baaf4e48SPavel Emelyanov port = xs_next_srcport(transport, port); 188867a391d7STrond Myklebust if (port > last) 188967a391d7STrond Myklebust nloop++; 189067a391d7STrond Myklebust } while (err == -EADDRINUSE && nloop != 2); 1891beb59b68SPavel Emelyanov 18924232e863SChuck Lever if (myaddr.ss_family == AF_INET) 1893beb59b68SPavel Emelyanov dprintk("RPC: %s %pI4:%u: %s (%d)\n", __func__, 1894beb59b68SPavel Emelyanov &((struct sockaddr_in *)&myaddr)->sin_addr, 1895beb59b68SPavel Emelyanov port, err ? "failed" : "ok", err); 1896beb59b68SPavel Emelyanov else 1897beb59b68SPavel Emelyanov dprintk("RPC: %s %pI6:%u: %s (%d)\n", __func__, 1898beb59b68SPavel Emelyanov &((struct sockaddr_in6 *)&myaddr)->sin6_addr, 18997dc753f0SChuck Lever port, err ? "failed" : "ok", err); 1900a246b010SChuck Lever return err; 1901a246b010SChuck Lever } 1902a246b010SChuck Lever 1903176e21eeSChuck Lever /* 1904176e21eeSChuck Lever * We don't support autobind on AF_LOCAL sockets 1905176e21eeSChuck Lever */ 1906176e21eeSChuck Lever static void xs_local_rpcbind(struct rpc_task *task) 1907176e21eeSChuck Lever { 1908fb43d172STrond Myklebust xprt_set_bound(task->tk_xprt); 1909176e21eeSChuck Lever } 1910176e21eeSChuck Lever 1911176e21eeSChuck Lever static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) 1912176e21eeSChuck Lever { 1913176e21eeSChuck Lever } 1914a246b010SChuck Lever 1915ed07536eSPeter Zijlstra #ifdef CONFIG_DEBUG_LOCK_ALLOC 1916ed07536eSPeter Zijlstra static struct lock_class_key xs_key[2]; 1917ed07536eSPeter Zijlstra static struct lock_class_key xs_slock_key[2]; 1918ed07536eSPeter Zijlstra 1919176e21eeSChuck Lever static inline void xs_reclassify_socketu(struct socket *sock) 1920176e21eeSChuck Lever { 1921176e21eeSChuck Lever struct sock *sk = sock->sk; 1922176e21eeSChuck Lever 1923176e21eeSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_LOCAL-RPC", 1924176e21eeSChuck Lever &xs_slock_key[1], "sk_lock-AF_LOCAL-RPC", &xs_key[1]); 1925176e21eeSChuck Lever } 1926176e21eeSChuck Lever 19278945ee5eSChuck Lever static inline void xs_reclassify_socket4(struct socket *sock) 1928ed07536eSPeter Zijlstra { 1929ed07536eSPeter Zijlstra struct sock *sk = sock->sk; 19308945ee5eSChuck Lever 19318945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET-RPC", 19328945ee5eSChuck Lever &xs_slock_key[0], "sk_lock-AF_INET-RPC", &xs_key[0]); 1933ed07536eSPeter Zijlstra } 19348945ee5eSChuck Lever 19358945ee5eSChuck Lever static inline void xs_reclassify_socket6(struct socket *sock) 19368945ee5eSChuck Lever { 19378945ee5eSChuck Lever struct sock *sk = sock->sk; 19388945ee5eSChuck Lever 19398945ee5eSChuck Lever sock_lock_init_class_and_name(sk, "slock-AF_INET6-RPC", 19408945ee5eSChuck Lever &xs_slock_key[1], "sk_lock-AF_INET6-RPC", &xs_key[1]); 1941ed07536eSPeter Zijlstra } 19426bc9638aSPavel Emelyanov 19436bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 19446bc9638aSPavel Emelyanov { 1945fafc4e1eSHannes Frederic Sowa if (WARN_ON_ONCE(!sock_allow_reclassification(sock->sk))) 19461b7a1819SWeston Andros Adamson return; 19471b7a1819SWeston Andros Adamson 19484232e863SChuck Lever switch (family) { 1949176e21eeSChuck Lever case AF_LOCAL: 1950176e21eeSChuck Lever xs_reclassify_socketu(sock); 1951176e21eeSChuck Lever break; 19524232e863SChuck Lever case AF_INET: 19536bc9638aSPavel Emelyanov xs_reclassify_socket4(sock); 19544232e863SChuck Lever break; 19554232e863SChuck Lever case AF_INET6: 19566bc9638aSPavel Emelyanov xs_reclassify_socket6(sock); 19574232e863SChuck Lever break; 19584232e863SChuck Lever } 19596bc9638aSPavel Emelyanov } 1960ed07536eSPeter Zijlstra #else 19616bc9638aSPavel Emelyanov static inline void xs_reclassify_socket(int family, struct socket *sock) 19626bc9638aSPavel Emelyanov { 19636bc9638aSPavel Emelyanov } 1964ed07536eSPeter Zijlstra #endif 1965ed07536eSPeter Zijlstra 196693dc41bdSNeilBrown static void xs_dummy_setup_socket(struct work_struct *work) 196793dc41bdSNeilBrown { 196893dc41bdSNeilBrown } 196993dc41bdSNeilBrown 19706bc9638aSPavel Emelyanov static struct socket *xs_create_sock(struct rpc_xprt *xprt, 19714dda9c8aSTrond Myklebust struct sock_xprt *transport, int family, int type, 19724dda9c8aSTrond Myklebust int protocol, bool reuseport) 197322f79326SPavel Emelyanov { 197422f79326SPavel Emelyanov struct socket *sock; 197522f79326SPavel Emelyanov int err; 197622f79326SPavel Emelyanov 19776bc9638aSPavel Emelyanov err = __sock_create(xprt->xprt_net, family, type, protocol, &sock, 1); 197822f79326SPavel Emelyanov if (err < 0) { 197922f79326SPavel Emelyanov dprintk("RPC: can't create %d transport socket (%d).\n", 198022f79326SPavel Emelyanov protocol, -err); 198122f79326SPavel Emelyanov goto out; 198222f79326SPavel Emelyanov } 19836bc9638aSPavel Emelyanov xs_reclassify_socket(family, sock); 198422f79326SPavel Emelyanov 19854dda9c8aSTrond Myklebust if (reuseport) 19864dda9c8aSTrond Myklebust xs_sock_set_reuseport(sock); 19874dda9c8aSTrond Myklebust 19884cea288aSBen Hutchings err = xs_bind(transport, sock); 19894cea288aSBen Hutchings if (err) { 199022f79326SPavel Emelyanov sock_release(sock); 199122f79326SPavel Emelyanov goto out; 199222f79326SPavel Emelyanov } 199322f79326SPavel Emelyanov 199422f79326SPavel Emelyanov return sock; 199522f79326SPavel Emelyanov out: 199622f79326SPavel Emelyanov return ERR_PTR(err); 199722f79326SPavel Emelyanov } 199822f79326SPavel Emelyanov 1999176e21eeSChuck Lever static int xs_local_finish_connecting(struct rpc_xprt *xprt, 2000176e21eeSChuck Lever struct socket *sock) 2001176e21eeSChuck Lever { 2002176e21eeSChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 2003176e21eeSChuck Lever xprt); 2004176e21eeSChuck Lever 2005176e21eeSChuck Lever if (!transport->inet) { 2006176e21eeSChuck Lever struct sock *sk = sock->sk; 2007176e21eeSChuck Lever 2008176e21eeSChuck Lever write_lock_bh(&sk->sk_callback_lock); 2009176e21eeSChuck Lever 2010176e21eeSChuck Lever xs_save_old_callbacks(transport, sk); 2011176e21eeSChuck Lever 2012176e21eeSChuck Lever sk->sk_user_data = xprt; 2013a2648094STrond Myklebust sk->sk_data_ready = xs_data_ready; 2014176e21eeSChuck Lever sk->sk_write_space = xs_udp_write_space; 2015b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 20162118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 2017c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 2018176e21eeSChuck Lever 2019176e21eeSChuck Lever xprt_clear_connected(xprt); 2020176e21eeSChuck Lever 2021176e21eeSChuck Lever /* Reset to new socket */ 2022176e21eeSChuck Lever transport->sock = sock; 2023176e21eeSChuck Lever transport->inet = sk; 2024176e21eeSChuck Lever 2025176e21eeSChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2026176e21eeSChuck Lever } 2027176e21eeSChuck Lever 20286c7a64e5STrond Myklebust transport->xmit.offset = 0; 20296c7a64e5STrond Myklebust 2030176e21eeSChuck Lever /* Tell the socket layer to start connecting... */ 2031176e21eeSChuck Lever xprt->stat.connect_count++; 2032176e21eeSChuck Lever xprt->stat.connect_start = jiffies; 2033176e21eeSChuck Lever return kernel_connect(sock, xs_addr(xprt), xprt->addrlen, 0); 2034176e21eeSChuck Lever } 2035176e21eeSChuck Lever 2036176e21eeSChuck Lever /** 2037176e21eeSChuck Lever * xs_local_setup_socket - create AF_LOCAL socket, connect to a local endpoint 2038176e21eeSChuck Lever * @transport: socket transport to connect 2039176e21eeSChuck Lever */ 2040dc107402SJ. Bruce Fields static int xs_local_setup_socket(struct sock_xprt *transport) 2041176e21eeSChuck Lever { 2042176e21eeSChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2043176e21eeSChuck Lever struct socket *sock; 2044176e21eeSChuck Lever int status = -EIO; 2045176e21eeSChuck Lever 2046176e21eeSChuck Lever status = __sock_create(xprt->xprt_net, AF_LOCAL, 2047176e21eeSChuck Lever SOCK_STREAM, 0, &sock, 1); 2048176e21eeSChuck Lever if (status < 0) { 2049176e21eeSChuck Lever dprintk("RPC: can't create AF_LOCAL " 2050176e21eeSChuck Lever "transport socket (%d).\n", -status); 2051176e21eeSChuck Lever goto out; 2052176e21eeSChuck Lever } 2053d1358917SStefan Hajnoczi xs_reclassify_socket(AF_LOCAL, sock); 2054176e21eeSChuck Lever 2055176e21eeSChuck Lever dprintk("RPC: worker connecting xprt %p via AF_LOCAL to %s\n", 2056176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 2057176e21eeSChuck Lever 2058176e21eeSChuck Lever status = xs_local_finish_connecting(xprt, sock); 205940b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2060176e21eeSChuck Lever switch (status) { 2061176e21eeSChuck Lever case 0: 2062176e21eeSChuck Lever dprintk("RPC: xprt %p connected to %s\n", 2063176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 2064176e21eeSChuck Lever xprt_set_connected(xprt); 20653601c4a9STrond Myklebust case -ENOBUFS: 2066176e21eeSChuck Lever break; 2067176e21eeSChuck Lever case -ENOENT: 2068176e21eeSChuck Lever dprintk("RPC: xprt %p: socket %s does not exist\n", 2069176e21eeSChuck Lever xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 2070176e21eeSChuck Lever break; 20714a20a988STrond Myklebust case -ECONNREFUSED: 20724a20a988STrond Myklebust dprintk("RPC: xprt %p: connection refused for %s\n", 20734a20a988STrond Myklebust xprt, xprt->address_strings[RPC_DISPLAY_ADDR]); 20744a20a988STrond Myklebust break; 2075176e21eeSChuck Lever default: 2076176e21eeSChuck Lever printk(KERN_ERR "%s: unhandled error (%d) connecting to %s\n", 2077176e21eeSChuck Lever __func__, -status, 2078176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2079176e21eeSChuck Lever } 2080176e21eeSChuck Lever 2081176e21eeSChuck Lever out: 2082176e21eeSChuck Lever xprt_clear_connecting(xprt); 2083176e21eeSChuck Lever xprt_wake_pending_tasks(xprt, status); 2084dc107402SJ. Bruce Fields return status; 2085dc107402SJ. Bruce Fields } 2086dc107402SJ. Bruce Fields 2087b6669737SLinus Torvalds static void xs_local_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2088dc107402SJ. Bruce Fields { 2089dc107402SJ. Bruce Fields struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2090dc107402SJ. Bruce Fields int ret; 2091dc107402SJ. Bruce Fields 2092dc107402SJ. Bruce Fields if (RPC_IS_ASYNC(task)) { 2093dc107402SJ. Bruce Fields /* 2094dc107402SJ. Bruce Fields * We want the AF_LOCAL connect to be resolved in the 2095dc107402SJ. Bruce Fields * filesystem namespace of the process making the rpc 2096dc107402SJ. Bruce Fields * call. Thus we connect synchronously. 2097dc107402SJ. Bruce Fields * 2098dc107402SJ. Bruce Fields * If we want to support asynchronous AF_LOCAL calls, 2099dc107402SJ. Bruce Fields * we'll need to figure out how to pass a namespace to 2100dc107402SJ. Bruce Fields * connect. 2101dc107402SJ. Bruce Fields */ 2102dc107402SJ. Bruce Fields rpc_exit(task, -ENOTCONN); 2103dc107402SJ. Bruce Fields return; 2104dc107402SJ. Bruce Fields } 2105dc107402SJ. Bruce Fields ret = xs_local_setup_socket(transport); 2106dc107402SJ. Bruce Fields if (ret && !RPC_IS_SOFTCONN(task)) 2107dc107402SJ. Bruce Fields msleep_interruptible(15000); 2108176e21eeSChuck Lever } 2109176e21eeSChuck Lever 21103c87ef6eSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_SWAP) 2111d6e971d8SJeff Layton /* 2112d6e971d8SJeff Layton * Note that this should be called with XPRT_LOCKED held (or when we otherwise 2113d6e971d8SJeff Layton * know that we have exclusive access to the socket), to guard against 2114d6e971d8SJeff Layton * races with xs_reset_transport. 2115d6e971d8SJeff Layton */ 2116a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2117a564b8f0SMel Gorman { 2118a564b8f0SMel Gorman struct sock_xprt *transport = container_of(xprt, struct sock_xprt, 2119a564b8f0SMel Gorman xprt); 2120a564b8f0SMel Gorman 2121d6e971d8SJeff Layton /* 2122d6e971d8SJeff Layton * If there's no sock, then we have nothing to set. The 2123d6e971d8SJeff Layton * reconnecting process will get it for us. 2124d6e971d8SJeff Layton */ 2125d6e971d8SJeff Layton if (!transport->inet) 2126d6e971d8SJeff Layton return; 21278e228133SJeff Layton if (atomic_read(&xprt->swapper)) 2128a564b8f0SMel Gorman sk_set_memalloc(transport->inet); 2129a564b8f0SMel Gorman } 2130a564b8f0SMel Gorman 2131a564b8f0SMel Gorman /** 2132d67fa4d8SJeff Layton * xs_enable_swap - Tag this transport as being used for swap. 2133a564b8f0SMel Gorman * @xprt: transport to tag 2134a564b8f0SMel Gorman * 21358e228133SJeff Layton * Take a reference to this transport on behalf of the rpc_clnt, and 21368e228133SJeff Layton * optionally mark it for swapping if it wasn't already. 2137a564b8f0SMel Gorman */ 2138d67fa4d8SJeff Layton static int 2139d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2140a564b8f0SMel Gorman { 2141d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 2142a564b8f0SMel Gorman 2143d6e971d8SJeff Layton if (atomic_inc_return(&xprt->swapper) != 1) 2144d6e971d8SJeff Layton return 0; 2145d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2146d6e971d8SJeff Layton return -ERESTARTSYS; 2147d6e971d8SJeff Layton if (xs->inet) 2148d6e971d8SJeff Layton sk_set_memalloc(xs->inet); 2149d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 21508e228133SJeff Layton return 0; 2151a564b8f0SMel Gorman } 2152a564b8f0SMel Gorman 21538e228133SJeff Layton /** 2154d67fa4d8SJeff Layton * xs_disable_swap - Untag this transport as being used for swap. 21558e228133SJeff Layton * @xprt: transport to tag 21568e228133SJeff Layton * 21578e228133SJeff Layton * Drop a "swapper" reference to this xprt on behalf of the rpc_clnt. If the 21588e228133SJeff Layton * swapper refcount goes to 0, untag the socket as a memalloc socket. 21598e228133SJeff Layton */ 2160d67fa4d8SJeff Layton static void 2161d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 21628e228133SJeff Layton { 2163d6e971d8SJeff Layton struct sock_xprt *xs = container_of(xprt, struct sock_xprt, xprt); 21648e228133SJeff Layton 2165d6e971d8SJeff Layton if (!atomic_dec_and_test(&xprt->swapper)) 2166d6e971d8SJeff Layton return; 2167d6e971d8SJeff Layton if (wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_KILLABLE)) 2168d6e971d8SJeff Layton return; 2169d6e971d8SJeff Layton if (xs->inet) 2170d6e971d8SJeff Layton sk_clear_memalloc(xs->inet); 2171d6e971d8SJeff Layton xprt_release_xprt(xprt, NULL); 2172a564b8f0SMel Gorman } 2173a564b8f0SMel Gorman #else 2174a564b8f0SMel Gorman static void xs_set_memalloc(struct rpc_xprt *xprt) 2175a564b8f0SMel Gorman { 2176a564b8f0SMel Gorman } 2177d67fa4d8SJeff Layton 2178d67fa4d8SJeff Layton static int 2179d67fa4d8SJeff Layton xs_enable_swap(struct rpc_xprt *xprt) 2180d67fa4d8SJeff Layton { 2181d67fa4d8SJeff Layton return -EINVAL; 2182d67fa4d8SJeff Layton } 2183d67fa4d8SJeff Layton 2184d67fa4d8SJeff Layton static void 2185d67fa4d8SJeff Layton xs_disable_swap(struct rpc_xprt *xprt) 2186d67fa4d8SJeff Layton { 2187d67fa4d8SJeff Layton } 2188a564b8f0SMel Gorman #endif 2189a564b8f0SMel Gorman 219016be2d20SChuck Lever static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 2191a246b010SChuck Lever { 219216be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2193edb267a6SChuck Lever 2194ee0ac0c2SChuck Lever if (!transport->inet) { 2195b0d93ad5SChuck Lever struct sock *sk = sock->sk; 2196b0d93ad5SChuck Lever 2197b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2198b0d93ad5SChuck Lever 21992a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 22002a9e1cfaSTrond Myklebust 2201b0d93ad5SChuck Lever sk->sk_user_data = xprt; 2202f9b2ee71STrond Myklebust sk->sk_data_ready = xs_data_ready; 2203b0d93ad5SChuck Lever sk->sk_write_space = xs_udp_write_space; 2204b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 2205c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 2206b0d93ad5SChuck Lever 2207b0d93ad5SChuck Lever xprt_set_connected(xprt); 2208b0d93ad5SChuck Lever 2209b0d93ad5SChuck Lever /* Reset to new socket */ 2210ee0ac0c2SChuck Lever transport->sock = sock; 2211ee0ac0c2SChuck Lever transport->inet = sk; 2212b0d93ad5SChuck Lever 2213a564b8f0SMel Gorman xs_set_memalloc(xprt); 2214a564b8f0SMel Gorman 2215b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2216b0d93ad5SChuck Lever } 2217470056c2SChuck Lever xs_udp_do_set_buffer_size(xprt); 221802910177STrond Myklebust 221902910177STrond Myklebust xprt->stat.connect_start = jiffies; 222016be2d20SChuck Lever } 222116be2d20SChuck Lever 22228c14ff2aSPavel Emelyanov static void xs_udp_setup_socket(struct work_struct *work) 2223a246b010SChuck Lever { 2224a246b010SChuck Lever struct sock_xprt *transport = 2225a246b010SChuck Lever container_of(work, struct sock_xprt, connect_worker.work); 2226a246b010SChuck Lever struct rpc_xprt *xprt = &transport->xprt; 2227d099b8afSColin Ian King struct socket *sock; 2228b65c0310SPavel Emelyanov int status = -EIO; 2229a246b010SChuck Lever 22308c14ff2aSPavel Emelyanov sock = xs_create_sock(xprt, transport, 22314dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_DGRAM, 22324dda9c8aSTrond Myklebust IPPROTO_UDP, false); 2233b65c0310SPavel Emelyanov if (IS_ERR(sock)) 2234a246b010SChuck Lever goto out; 223568e220bdSChuck Lever 2236c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2237c740eff8SChuck Lever "%s (port %s)\n", xprt, 2238c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2239c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2240c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 224168e220bdSChuck Lever 224268e220bdSChuck Lever xs_udp_finish_connecting(xprt, sock); 224340b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, 0); 2244a246b010SChuck Lever status = 0; 2245b0d93ad5SChuck Lever out: 2246718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 2247b0d93ad5SChuck Lever xprt_clear_connecting(xprt); 22487d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2249b0d93ad5SChuck Lever } 2250b0d93ad5SChuck Lever 22514876cc77STrond Myklebust /** 22524876cc77STrond Myklebust * xs_tcp_shutdown - gracefully shut down a TCP socket 22534876cc77STrond Myklebust * @xprt: transport 22544876cc77STrond Myklebust * 22554876cc77STrond Myklebust * Initiates a graceful shutdown of the TCP socket by calling the 22564876cc77STrond Myklebust * equivalent of shutdown(SHUT_RDWR); 22574876cc77STrond Myklebust */ 22584876cc77STrond Myklebust static void xs_tcp_shutdown(struct rpc_xprt *xprt) 22594876cc77STrond Myklebust { 22604876cc77STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 22614876cc77STrond Myklebust struct socket *sock = transport->sock; 22629b30889cSTrond Myklebust int skst = transport->inet ? transport->inet->sk_state : TCP_CLOSE; 22634876cc77STrond Myklebust 22644876cc77STrond Myklebust if (sock == NULL) 22654876cc77STrond Myklebust return; 22669b30889cSTrond Myklebust switch (skst) { 22679b30889cSTrond Myklebust default: 22684876cc77STrond Myklebust kernel_sock_shutdown(sock, SHUT_RDWR); 22694876cc77STrond Myklebust trace_rpc_socket_shutdown(xprt, sock); 22709b30889cSTrond Myklebust break; 22719b30889cSTrond Myklebust case TCP_CLOSE: 22729b30889cSTrond Myklebust case TCP_TIME_WAIT: 22734876cc77STrond Myklebust xs_reset_transport(transport); 22744876cc77STrond Myklebust } 22759b30889cSTrond Myklebust } 22764876cc77STrond Myklebust 22778d1b8c62STrond Myklebust static void xs_tcp_set_socket_timeouts(struct rpc_xprt *xprt, 22788d1b8c62STrond Myklebust struct socket *sock) 2279b0d93ad5SChuck Lever { 228016be2d20SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 22817196dbb0STrond Myklebust unsigned int keepidle; 22827196dbb0STrond Myklebust unsigned int keepcnt; 22837f260e85STrond Myklebust unsigned int opt_on = 1; 2284775f06abSTrond Myklebust unsigned int timeo; 22857f260e85STrond Myklebust 22867196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 22877196dbb0STrond Myklebust keepidle = DIV_ROUND_UP(xprt->timeout->to_initval, HZ); 22887196dbb0STrond Myklebust keepcnt = xprt->timeout->to_retries + 1; 22897196dbb0STrond Myklebust timeo = jiffies_to_msecs(xprt->timeout->to_initval) * 22907196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 22917196dbb0STrond Myklebust clear_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 22927196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 22937f260e85STrond Myklebust 22947f260e85STrond Myklebust /* TCP Keepalive options */ 22957f260e85STrond Myklebust kernel_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE, 22967f260e85STrond Myklebust (char *)&opt_on, sizeof(opt_on)); 22977f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPIDLE, 22987f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 22997f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPINTVL, 23007f260e85STrond Myklebust (char *)&keepidle, sizeof(keepidle)); 23017f260e85STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_KEEPCNT, 23027f260e85STrond Myklebust (char *)&keepcnt, sizeof(keepcnt)); 2303b0d93ad5SChuck Lever 23048d1b8c62STrond Myklebust /* TCP user timeout (see RFC5482) */ 23058d1b8c62STrond Myklebust kernel_setsockopt(sock, SOL_TCP, TCP_USER_TIMEOUT, 23068d1b8c62STrond Myklebust (char *)&timeo, sizeof(timeo)); 23078d1b8c62STrond Myklebust } 23088d1b8c62STrond Myklebust 23097196dbb0STrond Myklebust static void xs_tcp_set_connect_timeout(struct rpc_xprt *xprt, 23107196dbb0STrond Myklebust unsigned long connect_timeout, 23117196dbb0STrond Myklebust unsigned long reconnect_timeout) 23127196dbb0STrond Myklebust { 23137196dbb0STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 23147196dbb0STrond Myklebust struct rpc_timeout to; 23157196dbb0STrond Myklebust unsigned long initval; 23167196dbb0STrond Myklebust 23177196dbb0STrond Myklebust spin_lock_bh(&xprt->transport_lock); 23187196dbb0STrond Myklebust if (reconnect_timeout < xprt->max_reconnect_timeout) 23197196dbb0STrond Myklebust xprt->max_reconnect_timeout = reconnect_timeout; 23207196dbb0STrond Myklebust if (connect_timeout < xprt->connect_timeout) { 23217196dbb0STrond Myklebust memcpy(&to, xprt->timeout, sizeof(to)); 23227196dbb0STrond Myklebust initval = DIV_ROUND_UP(connect_timeout, to.to_retries + 1); 23237196dbb0STrond Myklebust /* Arbitrary lower limit */ 23247196dbb0STrond Myklebust if (initval < XS_TCP_INIT_REEST_TO << 1) 23257196dbb0STrond Myklebust initval = XS_TCP_INIT_REEST_TO << 1; 23267196dbb0STrond Myklebust to.to_initval = initval; 23277196dbb0STrond Myklebust to.to_maxval = initval; 23287196dbb0STrond Myklebust memcpy(&transport->tcp_timeout, &to, 23297196dbb0STrond Myklebust sizeof(transport->tcp_timeout)); 23307196dbb0STrond Myklebust xprt->timeout = &transport->tcp_timeout; 23317196dbb0STrond Myklebust xprt->connect_timeout = connect_timeout; 23327196dbb0STrond Myklebust } 23337196dbb0STrond Myklebust set_bit(XPRT_SOCK_UPD_TIMEOUT, &transport->sock_state); 23347196dbb0STrond Myklebust spin_unlock_bh(&xprt->transport_lock); 23357196dbb0STrond Myklebust } 23367196dbb0STrond Myklebust 23378d1b8c62STrond Myklebust static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) 23388d1b8c62STrond Myklebust { 23398d1b8c62STrond Myklebust struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 23408d1b8c62STrond Myklebust int ret = -ENOTCONN; 23418d1b8c62STrond Myklebust 23428d1b8c62STrond Myklebust if (!transport->inet) { 23438d1b8c62STrond Myklebust struct sock *sk = sock->sk; 23448d1b8c62STrond Myklebust unsigned int addr_pref = IPV6_PREFER_SRC_PUBLIC; 23458d1b8c62STrond Myklebust 2346d88e4d82SNeilBrown /* Avoid temporary address, they are bad for long-lived 2347d88e4d82SNeilBrown * connections such as NFS mounts. 2348d88e4d82SNeilBrown * RFC4941, section 3.6 suggests that: 2349d88e4d82SNeilBrown * Individual applications, which have specific 2350d88e4d82SNeilBrown * knowledge about the normal duration of connections, 2351d88e4d82SNeilBrown * MAY override this as appropriate. 2352d88e4d82SNeilBrown */ 2353d88e4d82SNeilBrown kernel_setsockopt(sock, SOL_IPV6, IPV6_ADDR_PREFERENCES, 2354d88e4d82SNeilBrown (char *)&addr_pref, sizeof(addr_pref)); 2355d88e4d82SNeilBrown 23568d1b8c62STrond Myklebust xs_tcp_set_socket_timeouts(xprt, sock); 2357775f06abSTrond Myklebust 2358b0d93ad5SChuck Lever write_lock_bh(&sk->sk_callback_lock); 2359b0d93ad5SChuck Lever 23602a9e1cfaSTrond Myklebust xs_save_old_callbacks(transport, sk); 23612a9e1cfaSTrond Myklebust 2362b0d93ad5SChuck Lever sk->sk_user_data = xprt; 23635157b956STrond Myklebust sk->sk_data_ready = xs_data_ready; 2364b0d93ad5SChuck Lever sk->sk_state_change = xs_tcp_state_change; 2365b0d93ad5SChuck Lever sk->sk_write_space = xs_tcp_write_space; 2366b4411457SEric Dumazet sock_set_flag(sk, SOCK_FASYNC); 23672118071dSTrond Myklebust sk->sk_error_report = xs_error_report; 2368c2126157STrond Myklebust sk->sk_allocation = GFP_NOIO; 23693167e12cSChuck Lever 23703167e12cSChuck Lever /* socket options */ 23713167e12cSChuck Lever sock_reset_flag(sk, SOCK_LINGER); 23723167e12cSChuck Lever tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF; 2373b0d93ad5SChuck Lever 2374b0d93ad5SChuck Lever xprt_clear_connected(xprt); 2375b0d93ad5SChuck Lever 2376b0d93ad5SChuck Lever /* Reset to new socket */ 2377ee0ac0c2SChuck Lever transport->sock = sock; 2378ee0ac0c2SChuck Lever transport->inet = sk; 2379b0d93ad5SChuck Lever 2380b0d93ad5SChuck Lever write_unlock_bh(&sk->sk_callback_lock); 2381b0d93ad5SChuck Lever } 2382b0d93ad5SChuck Lever 238301d37c42STrond Myklebust if (!xprt_bound(xprt)) 2384fe19a96bSTrond Myklebust goto out; 238501d37c42STrond Myklebust 2386a564b8f0SMel Gorman xs_set_memalloc(xprt); 2387a564b8f0SMel Gorman 2388e1806c7bSTrond Myklebust /* Reset TCP record info */ 2389e1806c7bSTrond Myklebust transport->recv.offset = 0; 2390e1806c7bSTrond Myklebust transport->recv.len = 0; 2391e1806c7bSTrond Myklebust transport->recv.copied = 0; 2392e1806c7bSTrond Myklebust transport->recv.flags = TCP_RCV_COPY_FRAGHDR | TCP_RCV_COPY_XID; 23936c7a64e5STrond Myklebust transport->xmit.offset = 0; 2394e1806c7bSTrond Myklebust 2395b0d93ad5SChuck Lever /* Tell the socket layer to start connecting... */ 2396262ca07dSChuck Lever xprt->stat.connect_count++; 2397262ca07dSChuck Lever xprt->stat.connect_start = jiffies; 23980fdea1e8STrond Myklebust set_bit(XPRT_SOCK_CONNECTING, &transport->sock_state); 2399fe19a96bSTrond Myklebust ret = kernel_connect(sock, xs_addr(xprt), xprt->addrlen, O_NONBLOCK); 2400fe19a96bSTrond Myklebust switch (ret) { 2401fe19a96bSTrond Myklebust case 0: 24024dda9c8aSTrond Myklebust xs_set_srcport(transport, sock); 2403e9d47639SGustavo A. R. Silva /* fall through */ 2404fe19a96bSTrond Myklebust case -EINPROGRESS: 2405fe19a96bSTrond Myklebust /* SYN_SENT! */ 2406fe19a96bSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 2407fe19a96bSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 24081f4c17a0STrond Myklebust break; 24091f4c17a0STrond Myklebust case -EADDRNOTAVAIL: 24101f4c17a0STrond Myklebust /* Source port number is unavailable. Try a new one! */ 24111f4c17a0STrond Myklebust transport->srcport = 0; 2412fe19a96bSTrond Myklebust } 2413fe19a96bSTrond Myklebust out: 2414fe19a96bSTrond Myklebust return ret; 241516be2d20SChuck Lever } 241616be2d20SChuck Lever 241716be2d20SChuck Lever /** 2418b61d59ffSTrond Myklebust * xs_tcp_setup_socket - create a TCP socket and connect to a remote endpoint 241916be2d20SChuck Lever * 242016be2d20SChuck Lever * Invoked by a work queue tasklet. 242116be2d20SChuck Lever */ 2422cdd518d5SPavel Emelyanov static void xs_tcp_setup_socket(struct work_struct *work) 242316be2d20SChuck Lever { 2424cdd518d5SPavel Emelyanov struct sock_xprt *transport = 2425cdd518d5SPavel Emelyanov container_of(work, struct sock_xprt, connect_worker.work); 242616be2d20SChuck Lever struct socket *sock = transport->sock; 2427a9f5f0f7SPavel Emelyanov struct rpc_xprt *xprt = &transport->xprt; 2428b61d59ffSTrond Myklebust int status = -EIO; 242916be2d20SChuck Lever 243016be2d20SChuck Lever if (!sock) { 2431cdd518d5SPavel Emelyanov sock = xs_create_sock(xprt, transport, 24324dda9c8aSTrond Myklebust xs_addr(xprt)->sa_family, SOCK_STREAM, 24334dda9c8aSTrond Myklebust IPPROTO_TCP, true); 2434b61d59ffSTrond Myklebust if (IS_ERR(sock)) { 2435b61d59ffSTrond Myklebust status = PTR_ERR(sock); 243616be2d20SChuck Lever goto out; 243716be2d20SChuck Lever } 24387d1e8255STrond Myklebust } 24397d1e8255STrond Myklebust 2440c740eff8SChuck Lever dprintk("RPC: worker connecting xprt %p via %s to " 2441c740eff8SChuck Lever "%s (port %s)\n", xprt, 2442c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO], 2443c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 2444c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT]); 244516be2d20SChuck Lever 244616be2d20SChuck Lever status = xs_tcp_finish_connecting(xprt, sock); 244740b5ea0cSTrond Myklebust trace_rpc_socket_connect(xprt, sock, status); 2448a246b010SChuck Lever dprintk("RPC: %p connect status %d connected %d sock state %d\n", 244946121cf7SChuck Lever xprt, -status, xprt_connected(xprt), 245046121cf7SChuck Lever sock->sk->sk_state); 2451a246b010SChuck Lever switch (status) { 2452f75e6745STrond Myklebust default: 2453f75e6745STrond Myklebust printk("%s: connect returned unhandled error %d\n", 2454f75e6745STrond Myklebust __func__, status); 2455e9d47639SGustavo A. R. Silva /* fall through */ 2456f75e6745STrond Myklebust case -EADDRNOTAVAIL: 2457f75e6745STrond Myklebust /* We're probably in TIME_WAIT. Get rid of existing socket, 2458f75e6745STrond Myklebust * and retry 2459f75e6745STrond Myklebust */ 2460a519fc7aSTrond Myklebust xs_tcp_force_close(xprt); 246188b5ed73STrond Myklebust break; 24622a491991STrond Myklebust case 0: 2463a246b010SChuck Lever case -EINPROGRESS: 2464a246b010SChuck Lever case -EALREADY: 2465718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 24667d1e8255STrond Myklebust return; 24679fcfe0c8STrond Myklebust case -EINVAL: 24689fcfe0c8STrond Myklebust /* Happens, for instance, if the user specified a link 24699fcfe0c8STrond Myklebust * local IPv6 address without a scope-id. 24709fcfe0c8STrond Myklebust */ 24713ed5e2a2STrond Myklebust case -ECONNREFUSED: 24723ed5e2a2STrond Myklebust case -ECONNRESET: 2473eb5b46faSTrond Myklebust case -ENETDOWN: 24743ed5e2a2STrond Myklebust case -ENETUNREACH: 24754ba161a7STrond Myklebust case -EHOSTUNREACH: 24763913c78cSTrond Myklebust case -EADDRINUSE: 24773601c4a9STrond Myklebust case -ENOBUFS: 24786ea44adcSNeilBrown /* 24796ea44adcSNeilBrown * xs_tcp_force_close() wakes tasks with -EIO. 24806ea44adcSNeilBrown * We need to wake them first to ensure the 24816ea44adcSNeilBrown * correct error code. 24826ea44adcSNeilBrown */ 24836ea44adcSNeilBrown xprt_wake_pending_tasks(xprt, status); 24844efdd92cSTrond Myklebust xs_tcp_force_close(xprt); 24859fcfe0c8STrond Myklebust goto out; 24868a2cec29STrond Myklebust } 24872a491991STrond Myklebust status = -EAGAIN; 2488a246b010SChuck Lever out: 2489718ba5b8STrond Myklebust xprt_unlock_connect(xprt, transport); 24902226feb6SChuck Lever xprt_clear_connecting(xprt); 24917d1e8255STrond Myklebust xprt_wake_pending_tasks(xprt, status); 2492a246b010SChuck Lever } 2493a246b010SChuck Lever 249402910177STrond Myklebust static unsigned long xs_reconnect_delay(const struct rpc_xprt *xprt) 249502910177STrond Myklebust { 249602910177STrond Myklebust unsigned long start, now = jiffies; 249702910177STrond Myklebust 249802910177STrond Myklebust start = xprt->stat.connect_start + xprt->reestablish_timeout; 249902910177STrond Myklebust if (time_after(start, now)) 250002910177STrond Myklebust return start - now; 250102910177STrond Myklebust return 0; 250202910177STrond Myklebust } 250302910177STrond Myklebust 25043851f1cdSTrond Myklebust static void xs_reconnect_backoff(struct rpc_xprt *xprt) 25053851f1cdSTrond Myklebust { 25063851f1cdSTrond Myklebust xprt->reestablish_timeout <<= 1; 25073851f1cdSTrond Myklebust if (xprt->reestablish_timeout > xprt->max_reconnect_timeout) 25083851f1cdSTrond Myklebust xprt->reestablish_timeout = xprt->max_reconnect_timeout; 25093851f1cdSTrond Myklebust if (xprt->reestablish_timeout < XS_TCP_INIT_REEST_TO) 25103851f1cdSTrond Myklebust xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 25113851f1cdSTrond Myklebust } 25123851f1cdSTrond Myklebust 251368e220bdSChuck Lever /** 25149903cd1cSChuck Lever * xs_connect - connect a socket to a remote endpoint 25151b092092STrond Myklebust * @xprt: pointer to transport structure 25169903cd1cSChuck Lever * @task: address of RPC task that manages state of connect request 25179903cd1cSChuck Lever * 25189903cd1cSChuck Lever * TCP: If the remote end dropped the connection, delay reconnecting. 251903bf4b70SChuck Lever * 252003bf4b70SChuck Lever * UDP socket connects are synchronous, but we use a work queue anyway 252103bf4b70SChuck Lever * to guarantee that even unprivileged user processes can set up a 252203bf4b70SChuck Lever * socket on a privileged port. 252303bf4b70SChuck Lever * 252403bf4b70SChuck Lever * If a UDP socket connect fails, the delay behavior here prevents 252503bf4b70SChuck Lever * retry floods (hard mounts). 25269903cd1cSChuck Lever */ 25271b092092STrond Myklebust static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) 2528a246b010SChuck Lever { 2529ee0ac0c2SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 253002910177STrond Myklebust unsigned long delay = 0; 2531a246b010SChuck Lever 2532718ba5b8STrond Myklebust WARN_ON_ONCE(!xprt_lock_connect(xprt, task, transport)); 2533718ba5b8STrond Myklebust 253499b1a4c3STrond Myklebust if (transport->sock != NULL) { 253546121cf7SChuck Lever dprintk("RPC: xs_connect delayed xprt %p for %lu " 253646121cf7SChuck Lever "seconds\n", 253703bf4b70SChuck Lever xprt, xprt->reestablish_timeout / HZ); 253899b1a4c3STrond Myklebust 253999b1a4c3STrond Myklebust /* Start by resetting any existing state */ 254099b1a4c3STrond Myklebust xs_reset_transport(transport); 254199b1a4c3STrond Myklebust 254202910177STrond Myklebust delay = xs_reconnect_delay(xprt); 25433851f1cdSTrond Myklebust xs_reconnect_backoff(xprt); 254402910177STrond Myklebust 254502910177STrond Myklebust } else 25469903cd1cSChuck Lever dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); 254702910177STrond Myklebust 254840a5f1b1STrond Myklebust queue_delayed_work(xprtiod_workqueue, 254902910177STrond Myklebust &transport->connect_worker, 255002910177STrond Myklebust delay); 2551a246b010SChuck Lever } 2552a246b010SChuck Lever 2553262ca07dSChuck Lever /** 2554176e21eeSChuck Lever * xs_local_print_stats - display AF_LOCAL socket-specifc stats 2555176e21eeSChuck Lever * @xprt: rpc_xprt struct containing statistics 2556176e21eeSChuck Lever * @seq: output file 2557176e21eeSChuck Lever * 2558176e21eeSChuck Lever */ 2559176e21eeSChuck Lever static void xs_local_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2560176e21eeSChuck Lever { 2561176e21eeSChuck Lever long idle_time = 0; 2562176e21eeSChuck Lever 2563176e21eeSChuck Lever if (xprt_connected(xprt)) 2564176e21eeSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2565176e21eeSChuck Lever 2566176e21eeSChuck Lever seq_printf(seq, "\txprt:\tlocal %lu %lu %lu %ld %lu %lu %lu " 256715a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2568176e21eeSChuck Lever xprt->stat.bind_count, 2569176e21eeSChuck Lever xprt->stat.connect_count, 2570176e21eeSChuck Lever xprt->stat.connect_time, 2571176e21eeSChuck Lever idle_time, 2572176e21eeSChuck Lever xprt->stat.sends, 2573176e21eeSChuck Lever xprt->stat.recvs, 2574176e21eeSChuck Lever xprt->stat.bad_xids, 2575176e21eeSChuck Lever xprt->stat.req_u, 257615a45206SAndy Adamson xprt->stat.bklog_u, 257715a45206SAndy Adamson xprt->stat.max_slots, 257815a45206SAndy Adamson xprt->stat.sending_u, 257915a45206SAndy Adamson xprt->stat.pending_u); 2580176e21eeSChuck Lever } 2581176e21eeSChuck Lever 2582176e21eeSChuck Lever /** 2583262ca07dSChuck Lever * xs_udp_print_stats - display UDP socket-specifc stats 2584262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2585262ca07dSChuck Lever * @seq: output file 2586262ca07dSChuck Lever * 2587262ca07dSChuck Lever */ 2588262ca07dSChuck Lever static void xs_udp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2589262ca07dSChuck Lever { 2590c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2591c8475461SChuck Lever 259215a45206SAndy Adamson seq_printf(seq, "\txprt:\tudp %u %lu %lu %lu %lu %llu %llu " 259315a45206SAndy Adamson "%lu %llu %llu\n", 2594fbfffbd5SChuck Lever transport->srcport, 2595262ca07dSChuck Lever xprt->stat.bind_count, 2596262ca07dSChuck Lever xprt->stat.sends, 2597262ca07dSChuck Lever xprt->stat.recvs, 2598262ca07dSChuck Lever xprt->stat.bad_xids, 2599262ca07dSChuck Lever xprt->stat.req_u, 260015a45206SAndy Adamson xprt->stat.bklog_u, 260115a45206SAndy Adamson xprt->stat.max_slots, 260215a45206SAndy Adamson xprt->stat.sending_u, 260315a45206SAndy Adamson xprt->stat.pending_u); 2604262ca07dSChuck Lever } 2605262ca07dSChuck Lever 2606262ca07dSChuck Lever /** 2607262ca07dSChuck Lever * xs_tcp_print_stats - display TCP socket-specifc stats 2608262ca07dSChuck Lever * @xprt: rpc_xprt struct containing statistics 2609262ca07dSChuck Lever * @seq: output file 2610262ca07dSChuck Lever * 2611262ca07dSChuck Lever */ 2612262ca07dSChuck Lever static void xs_tcp_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) 2613262ca07dSChuck Lever { 2614c8475461SChuck Lever struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); 2615262ca07dSChuck Lever long idle_time = 0; 2616262ca07dSChuck Lever 2617262ca07dSChuck Lever if (xprt_connected(xprt)) 2618262ca07dSChuck Lever idle_time = (long)(jiffies - xprt->last_used) / HZ; 2619262ca07dSChuck Lever 262015a45206SAndy Adamson seq_printf(seq, "\txprt:\ttcp %u %lu %lu %lu %ld %lu %lu %lu " 262115a45206SAndy Adamson "%llu %llu %lu %llu %llu\n", 2622fbfffbd5SChuck Lever transport->srcport, 2623262ca07dSChuck Lever xprt->stat.bind_count, 2624262ca07dSChuck Lever xprt->stat.connect_count, 2625262ca07dSChuck Lever xprt->stat.connect_time, 2626262ca07dSChuck Lever idle_time, 2627262ca07dSChuck Lever xprt->stat.sends, 2628262ca07dSChuck Lever xprt->stat.recvs, 2629262ca07dSChuck Lever xprt->stat.bad_xids, 2630262ca07dSChuck Lever xprt->stat.req_u, 263115a45206SAndy Adamson xprt->stat.bklog_u, 263215a45206SAndy Adamson xprt->stat.max_slots, 263315a45206SAndy Adamson xprt->stat.sending_u, 263415a45206SAndy Adamson xprt->stat.pending_u); 2635262ca07dSChuck Lever } 2636262ca07dSChuck Lever 26374cfc7e60SRahul Iyer /* 26384cfc7e60SRahul Iyer * Allocate a bunch of pages for a scratch buffer for the rpc code. The reason 26394cfc7e60SRahul Iyer * we allocate pages instead doing a kmalloc like rpc_malloc is because we want 26404cfc7e60SRahul Iyer * to use the server side send routines. 26414cfc7e60SRahul Iyer */ 26425fe6eaa1SChuck Lever static int bc_malloc(struct rpc_task *task) 26434cfc7e60SRahul Iyer { 26445fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp; 26455fe6eaa1SChuck Lever size_t size = rqst->rq_callsize; 26464cfc7e60SRahul Iyer struct page *page; 26474cfc7e60SRahul Iyer struct rpc_buffer *buf; 26484cfc7e60SRahul Iyer 26495fe6eaa1SChuck Lever if (size > PAGE_SIZE - sizeof(struct rpc_buffer)) { 26505fe6eaa1SChuck Lever WARN_ONCE(1, "xprtsock: large bc buffer request (size %zu)\n", 26515fe6eaa1SChuck Lever size); 26525fe6eaa1SChuck Lever return -EINVAL; 26535fe6eaa1SChuck Lever } 26544cfc7e60SRahul Iyer 2655b8a13d03SWeston Andros Adamson page = alloc_page(GFP_KERNEL); 26564cfc7e60SRahul Iyer if (!page) 26575fe6eaa1SChuck Lever return -ENOMEM; 26584cfc7e60SRahul Iyer 26594cfc7e60SRahul Iyer buf = page_address(page); 26604cfc7e60SRahul Iyer buf->len = PAGE_SIZE; 26614cfc7e60SRahul Iyer 26625fe6eaa1SChuck Lever rqst->rq_buffer = buf->data; 266318e601d6SJeff Layton rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; 26645fe6eaa1SChuck Lever return 0; 26654cfc7e60SRahul Iyer } 26664cfc7e60SRahul Iyer 26674cfc7e60SRahul Iyer /* 26684cfc7e60SRahul Iyer * Free the space allocated in the bc_alloc routine 26694cfc7e60SRahul Iyer */ 26703435c74aSChuck Lever static void bc_free(struct rpc_task *task) 26714cfc7e60SRahul Iyer { 26723435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer; 26734cfc7e60SRahul Iyer struct rpc_buffer *buf; 26744cfc7e60SRahul Iyer 26754cfc7e60SRahul Iyer buf = container_of(buffer, struct rpc_buffer, data); 26764cfc7e60SRahul Iyer free_page((unsigned long)buf); 26774cfc7e60SRahul Iyer } 26784cfc7e60SRahul Iyer 26794cfc7e60SRahul Iyer /* 26804cfc7e60SRahul Iyer * Use the svc_sock to send the callback. Must be called with svsk->sk_mutex 26814cfc7e60SRahul Iyer * held. Borrows heavily from svc_tcp_sendto and xs_tcp_send_request. 26824cfc7e60SRahul Iyer */ 26834cfc7e60SRahul Iyer static int bc_sendto(struct rpc_rqst *req) 26844cfc7e60SRahul Iyer { 26854cfc7e60SRahul Iyer int len; 26864cfc7e60SRahul Iyer struct xdr_buf *xbufp = &req->rq_snd_buf; 26874cfc7e60SRahul Iyer struct rpc_xprt *xprt = req->rq_xprt; 26884cfc7e60SRahul Iyer struct sock_xprt *transport = 26894cfc7e60SRahul Iyer container_of(xprt, struct sock_xprt, xprt); 26904cfc7e60SRahul Iyer struct socket *sock = transport->sock; 26914cfc7e60SRahul Iyer unsigned long headoff; 26924cfc7e60SRahul Iyer unsigned long tailoff; 26934cfc7e60SRahul Iyer 269461677eeeSChuck Lever xs_encode_stream_record_marker(xbufp); 26954cfc7e60SRahul Iyer 26964cfc7e60SRahul Iyer tailoff = (unsigned long)xbufp->tail[0].iov_base & ~PAGE_MASK; 26974cfc7e60SRahul Iyer headoff = (unsigned long)xbufp->head[0].iov_base & ~PAGE_MASK; 26984cfc7e60SRahul Iyer len = svc_send_common(sock, xbufp, 26994cfc7e60SRahul Iyer virt_to_page(xbufp->head[0].iov_base), headoff, 27004cfc7e60SRahul Iyer xbufp->tail[0].iov_base, tailoff); 27014cfc7e60SRahul Iyer 27024cfc7e60SRahul Iyer if (len != xbufp->len) { 27034cfc7e60SRahul Iyer printk(KERN_NOTICE "Error sending entire callback!\n"); 27044cfc7e60SRahul Iyer len = -EAGAIN; 27054cfc7e60SRahul Iyer } 27064cfc7e60SRahul Iyer 27074cfc7e60SRahul Iyer return len; 27084cfc7e60SRahul Iyer } 27094cfc7e60SRahul Iyer 27104cfc7e60SRahul Iyer /* 27114cfc7e60SRahul Iyer * The send routine. Borrows from svc_send 27124cfc7e60SRahul Iyer */ 271350f484e2STrond Myklebust static int bc_send_request(struct rpc_rqst *req, struct rpc_task *task) 27144cfc7e60SRahul Iyer { 27154cfc7e60SRahul Iyer struct svc_xprt *xprt; 27167fc56136SAndrzej Hajda int len; 27174cfc7e60SRahul Iyer 27184cfc7e60SRahul Iyer dprintk("sending request with xid: %08x\n", ntohl(req->rq_xid)); 27194cfc7e60SRahul Iyer /* 27204cfc7e60SRahul Iyer * Get the server socket associated with this callback xprt 27214cfc7e60SRahul Iyer */ 27224cfc7e60SRahul Iyer xprt = req->rq_xprt->bc_xprt; 27234cfc7e60SRahul Iyer 27244cfc7e60SRahul Iyer /* 27254cfc7e60SRahul Iyer * Grab the mutex to serialize data as the connection is shared 27264cfc7e60SRahul Iyer * with the fore channel 27274cfc7e60SRahul Iyer */ 27284cfc7e60SRahul Iyer if (!mutex_trylock(&xprt->xpt_mutex)) { 27294cfc7e60SRahul Iyer rpc_sleep_on(&xprt->xpt_bc_pending, task, NULL); 27304cfc7e60SRahul Iyer if (!mutex_trylock(&xprt->xpt_mutex)) 27314cfc7e60SRahul Iyer return -EAGAIN; 27324cfc7e60SRahul Iyer rpc_wake_up_queued_task(&xprt->xpt_bc_pending, task); 27334cfc7e60SRahul Iyer } 27344cfc7e60SRahul Iyer if (test_bit(XPT_DEAD, &xprt->xpt_flags)) 27354cfc7e60SRahul Iyer len = -ENOTCONN; 27364cfc7e60SRahul Iyer else 27374cfc7e60SRahul Iyer len = bc_sendto(req); 27384cfc7e60SRahul Iyer mutex_unlock(&xprt->xpt_mutex); 27394cfc7e60SRahul Iyer 27404cfc7e60SRahul Iyer if (len > 0) 27414cfc7e60SRahul Iyer len = 0; 27424cfc7e60SRahul Iyer 27434cfc7e60SRahul Iyer return len; 27444cfc7e60SRahul Iyer } 27454cfc7e60SRahul Iyer 27464cfc7e60SRahul Iyer /* 27474cfc7e60SRahul Iyer * The close routine. Since this is client initiated, we do nothing 27484cfc7e60SRahul Iyer */ 27494cfc7e60SRahul Iyer 27504cfc7e60SRahul Iyer static void bc_close(struct rpc_xprt *xprt) 27514cfc7e60SRahul Iyer { 27524cfc7e60SRahul Iyer } 27534cfc7e60SRahul Iyer 27544cfc7e60SRahul Iyer /* 27554cfc7e60SRahul Iyer * The xprt destroy routine. Again, because this connection is client 27564cfc7e60SRahul Iyer * initiated, we do nothing 27574cfc7e60SRahul Iyer */ 27584cfc7e60SRahul Iyer 27594cfc7e60SRahul Iyer static void bc_destroy(struct rpc_xprt *xprt) 27604cfc7e60SRahul Iyer { 276147f72efaSKinglong Mee dprintk("RPC: bc_destroy xprt %p\n", xprt); 276247f72efaSKinglong Mee 276347f72efaSKinglong Mee xs_xprt_free(xprt); 276447f72efaSKinglong Mee module_put(THIS_MODULE); 27654cfc7e60SRahul Iyer } 27664cfc7e60SRahul Iyer 2767d31ae254SChuck Lever static const struct rpc_xprt_ops xs_local_ops = { 2768176e21eeSChuck Lever .reserve_xprt = xprt_reserve_xprt, 27694cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 2770f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2771a9cde23aSChuck Lever .free_slot = xprt_free_slot, 2772176e21eeSChuck Lever .rpcbind = xs_local_rpcbind, 2773176e21eeSChuck Lever .set_port = xs_local_set_port, 2774dc107402SJ. Bruce Fields .connect = xs_local_connect, 2775176e21eeSChuck Lever .buf_alloc = rpc_malloc, 2776176e21eeSChuck Lever .buf_free = rpc_free, 2777176e21eeSChuck Lever .send_request = xs_local_send_request, 2778176e21eeSChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2779176e21eeSChuck Lever .close = xs_close, 2780a1311d87STrond Myklebust .destroy = xs_destroy, 2781176e21eeSChuck Lever .print_stats = xs_local_print_stats, 2782d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2783d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 2784176e21eeSChuck Lever }; 2785176e21eeSChuck Lever 2786d31ae254SChuck Lever static const struct rpc_xprt_ops xs_udp_ops = { 278743118c29SChuck Lever .set_buffer_size = xs_udp_set_buffer_size, 278812a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt_cong, 278949e9a890SChuck Lever .release_xprt = xprt_release_xprt_cong, 2790f39c1bfbSTrond Myklebust .alloc_slot = xprt_alloc_slot, 2791a9cde23aSChuck Lever .free_slot = xprt_free_slot, 279245160d62SChuck Lever .rpcbind = rpcb_getport_async, 279392200412SChuck Lever .set_port = xs_set_port, 27949903cd1cSChuck Lever .connect = xs_connect, 279502107148SChuck Lever .buf_alloc = rpc_malloc, 279602107148SChuck Lever .buf_free = rpc_free, 2797262965f5SChuck Lever .send_request = xs_udp_send_request, 2798fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_rtt, 279946c0ee8bSChuck Lever .timer = xs_udp_timer, 2800a58dd398SChuck Lever .release_request = xprt_release_rqst_cong, 2801262965f5SChuck Lever .close = xs_close, 2802262965f5SChuck Lever .destroy = xs_destroy, 2803262ca07dSChuck Lever .print_stats = xs_udp_print_stats, 2804d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2805d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 28064a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 2807262965f5SChuck Lever }; 2808262965f5SChuck Lever 2809d31ae254SChuck Lever static const struct rpc_xprt_ops xs_tcp_ops = { 281012a80469SChuck Lever .reserve_xprt = xprt_reserve_xprt, 28114cd34e7cSTrond Myklebust .release_xprt = xprt_release_xprt, 2812*36bd7de9STrond Myklebust .alloc_slot = xprt_alloc_slot, 2813a9cde23aSChuck Lever .free_slot = xprt_free_slot, 281445160d62SChuck Lever .rpcbind = rpcb_getport_async, 281592200412SChuck Lever .set_port = xs_set_port, 28160b9e7943STrond Myklebust .connect = xs_connect, 281702107148SChuck Lever .buf_alloc = rpc_malloc, 281802107148SChuck Lever .buf_free = rpc_free, 2819262965f5SChuck Lever .send_request = xs_tcp_send_request, 2820fe3aca29SChuck Lever .set_retrans_timeout = xprt_set_retrans_timeout_def, 2821c627d31bSTrond Myklebust .close = xs_tcp_shutdown, 28229903cd1cSChuck Lever .destroy = xs_destroy, 28237196dbb0STrond Myklebust .set_connect_timeout = xs_tcp_set_connect_timeout, 2824262ca07dSChuck Lever .print_stats = xs_tcp_print_stats, 2825d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2826d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 28274a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 282842e5c3e2SChuck Lever #ifdef CONFIG_SUNRPC_BACKCHANNEL 282942e5c3e2SChuck Lever .bc_setup = xprt_setup_bc, 283076566773SChuck Lever .bc_up = xs_tcp_bc_up, 28316b26cc8cSChuck Lever .bc_maxpayload = xs_tcp_bc_maxpayload, 283242e5c3e2SChuck Lever .bc_free_rqst = xprt_free_bc_rqst, 283342e5c3e2SChuck Lever .bc_destroy = xprt_destroy_bc, 283442e5c3e2SChuck Lever #endif 2835a246b010SChuck Lever }; 2836a246b010SChuck Lever 28374cfc7e60SRahul Iyer /* 28384cfc7e60SRahul Iyer * The rpc_xprt_ops for the server backchannel 28394cfc7e60SRahul Iyer */ 28404cfc7e60SRahul Iyer 2841d31ae254SChuck Lever static const struct rpc_xprt_ops bc_tcp_ops = { 28424cfc7e60SRahul Iyer .reserve_xprt = xprt_reserve_xprt, 28434cfc7e60SRahul Iyer .release_xprt = xprt_release_xprt, 284484e28a30SBryan Schumaker .alloc_slot = xprt_alloc_slot, 2845a9cde23aSChuck Lever .free_slot = xprt_free_slot, 28464cfc7e60SRahul Iyer .buf_alloc = bc_malloc, 28474cfc7e60SRahul Iyer .buf_free = bc_free, 28484cfc7e60SRahul Iyer .send_request = bc_send_request, 28494cfc7e60SRahul Iyer .set_retrans_timeout = xprt_set_retrans_timeout_def, 28504cfc7e60SRahul Iyer .close = bc_close, 28514cfc7e60SRahul Iyer .destroy = bc_destroy, 28524cfc7e60SRahul Iyer .print_stats = xs_tcp_print_stats, 2853d67fa4d8SJeff Layton .enable_swap = xs_enable_swap, 2854d67fa4d8SJeff Layton .disable_swap = xs_disable_swap, 28554a068258SChuck Lever .inject_disconnect = xs_inject_disconnect, 28564cfc7e60SRahul Iyer }; 28574cfc7e60SRahul Iyer 285892476850SChuck Lever static int xs_init_anyaddr(const int family, struct sockaddr *sap) 285992476850SChuck Lever { 286092476850SChuck Lever static const struct sockaddr_in sin = { 286192476850SChuck Lever .sin_family = AF_INET, 286292476850SChuck Lever .sin_addr.s_addr = htonl(INADDR_ANY), 286392476850SChuck Lever }; 286492476850SChuck Lever static const struct sockaddr_in6 sin6 = { 286592476850SChuck Lever .sin6_family = AF_INET6, 286692476850SChuck Lever .sin6_addr = IN6ADDR_ANY_INIT, 286792476850SChuck Lever }; 286892476850SChuck Lever 286992476850SChuck Lever switch (family) { 2870176e21eeSChuck Lever case AF_LOCAL: 2871176e21eeSChuck Lever break; 287292476850SChuck Lever case AF_INET: 287392476850SChuck Lever memcpy(sap, &sin, sizeof(sin)); 287492476850SChuck Lever break; 287592476850SChuck Lever case AF_INET6: 287692476850SChuck Lever memcpy(sap, &sin6, sizeof(sin6)); 287792476850SChuck Lever break; 287892476850SChuck Lever default: 287992476850SChuck Lever dprintk("RPC: %s: Bad address family\n", __func__); 288092476850SChuck Lever return -EAFNOSUPPORT; 288192476850SChuck Lever } 288292476850SChuck Lever return 0; 288392476850SChuck Lever } 288492476850SChuck Lever 28853c341b0bS\"Talpey, Thomas\ static struct rpc_xprt *xs_setup_xprt(struct xprt_create *args, 2886d9ba131dSTrond Myklebust unsigned int slot_table_size, 2887d9ba131dSTrond Myklebust unsigned int max_slot_table_size) 2888c8541ecdSChuck Lever { 2889c8541ecdSChuck Lever struct rpc_xprt *xprt; 2890ffc2e518SChuck Lever struct sock_xprt *new; 2891c8541ecdSChuck Lever 289296802a09SFrank van Maarseveen if (args->addrlen > sizeof(xprt->addr)) { 2893c8541ecdSChuck Lever dprintk("RPC: xs_setup_xprt: address too large\n"); 2894c8541ecdSChuck Lever return ERR_PTR(-EBADF); 2895c8541ecdSChuck Lever } 2896c8541ecdSChuck Lever 2897d9ba131dSTrond Myklebust xprt = xprt_alloc(args->net, sizeof(*new), slot_table_size, 2898d9ba131dSTrond Myklebust max_slot_table_size); 2899bd1722d4SPavel Emelyanov if (xprt == NULL) { 290046121cf7SChuck Lever dprintk("RPC: xs_setup_xprt: couldn't allocate " 290146121cf7SChuck Lever "rpc_xprt\n"); 2902c8541ecdSChuck Lever return ERR_PTR(-ENOMEM); 2903c8541ecdSChuck Lever } 2904c8541ecdSChuck Lever 2905bd1722d4SPavel Emelyanov new = container_of(xprt, struct sock_xprt, xprt); 2906edc1b01cSTrond Myklebust mutex_init(&new->recv_mutex); 290796802a09SFrank van Maarseveen memcpy(&xprt->addr, args->dstaddr, args->addrlen); 290896802a09SFrank van Maarseveen xprt->addrlen = args->addrlen; 2909d3bc9a1dSFrank van Maarseveen if (args->srcaddr) 2910fbfffbd5SChuck Lever memcpy(&new->srcaddr, args->srcaddr, args->addrlen); 291192476850SChuck Lever else { 291292476850SChuck Lever int err; 291392476850SChuck Lever err = xs_init_anyaddr(args->dstaddr->sa_family, 291492476850SChuck Lever (struct sockaddr *)&new->srcaddr); 29152aa13531SStanislav Kinsbursky if (err != 0) { 29162aa13531SStanislav Kinsbursky xprt_free(xprt); 291792476850SChuck Lever return ERR_PTR(err); 291892476850SChuck Lever } 29192aa13531SStanislav Kinsbursky } 2920c8541ecdSChuck Lever 2921c8541ecdSChuck Lever return xprt; 2922c8541ecdSChuck Lever } 2923c8541ecdSChuck Lever 2924176e21eeSChuck Lever static const struct rpc_timeout xs_local_default_timeout = { 2925176e21eeSChuck Lever .to_initval = 10 * HZ, 2926176e21eeSChuck Lever .to_maxval = 10 * HZ, 2927176e21eeSChuck Lever .to_retries = 2, 2928176e21eeSChuck Lever }; 2929176e21eeSChuck Lever 2930176e21eeSChuck Lever /** 2931176e21eeSChuck Lever * xs_setup_local - Set up transport to use an AF_LOCAL socket 2932176e21eeSChuck Lever * @args: rpc transport creation arguments 2933176e21eeSChuck Lever * 2934176e21eeSChuck Lever * AF_LOCAL is a "tpi_cots_ord" transport, just like TCP 2935176e21eeSChuck Lever */ 2936176e21eeSChuck Lever static struct rpc_xprt *xs_setup_local(struct xprt_create *args) 2937176e21eeSChuck Lever { 2938176e21eeSChuck Lever struct sockaddr_un *sun = (struct sockaddr_un *)args->dstaddr; 2939176e21eeSChuck Lever struct sock_xprt *transport; 2940176e21eeSChuck Lever struct rpc_xprt *xprt; 2941176e21eeSChuck Lever struct rpc_xprt *ret; 2942176e21eeSChuck Lever 2943d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 2944d9ba131dSTrond Myklebust xprt_max_tcp_slot_table_entries); 2945176e21eeSChuck Lever if (IS_ERR(xprt)) 2946176e21eeSChuck Lever return xprt; 2947176e21eeSChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 2948176e21eeSChuck Lever 2949176e21eeSChuck Lever xprt->prot = 0; 2950176e21eeSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 2951176e21eeSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 2952176e21eeSChuck Lever 2953176e21eeSChuck Lever xprt->bind_timeout = XS_BIND_TO; 2954176e21eeSChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 2955176e21eeSChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 2956176e21eeSChuck Lever 2957176e21eeSChuck Lever xprt->ops = &xs_local_ops; 2958176e21eeSChuck Lever xprt->timeout = &xs_local_default_timeout; 2959176e21eeSChuck Lever 2960a2648094STrond Myklebust INIT_WORK(&transport->recv_worker, xs_local_data_receive_workfn); 296193dc41bdSNeilBrown INIT_DELAYED_WORK(&transport->connect_worker, 296293dc41bdSNeilBrown xs_dummy_setup_socket); 296393dc41bdSNeilBrown 2964176e21eeSChuck Lever switch (sun->sun_family) { 2965176e21eeSChuck Lever case AF_LOCAL: 2966176e21eeSChuck Lever if (sun->sun_path[0] != '/') { 2967176e21eeSChuck Lever dprintk("RPC: bad AF_LOCAL address: %s\n", 2968176e21eeSChuck Lever sun->sun_path); 2969176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2970176e21eeSChuck Lever goto out_err; 2971176e21eeSChuck Lever } 2972176e21eeSChuck Lever xprt_set_bound(xprt); 2973176e21eeSChuck Lever xs_format_peer_addresses(xprt, "local", RPCBIND_NETID_LOCAL); 29747073ea87SJ. Bruce Fields ret = ERR_PTR(xs_local_setup_socket(transport)); 29757073ea87SJ. Bruce Fields if (ret) 29767073ea87SJ. Bruce Fields goto out_err; 2977176e21eeSChuck Lever break; 2978176e21eeSChuck Lever default: 2979176e21eeSChuck Lever ret = ERR_PTR(-EAFNOSUPPORT); 2980176e21eeSChuck Lever goto out_err; 2981176e21eeSChuck Lever } 2982176e21eeSChuck Lever 2983176e21eeSChuck Lever dprintk("RPC: set up xprt to %s via AF_LOCAL\n", 2984176e21eeSChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR]); 2985176e21eeSChuck Lever 2986176e21eeSChuck Lever if (try_module_get(THIS_MODULE)) 2987176e21eeSChuck Lever return xprt; 2988176e21eeSChuck Lever ret = ERR_PTR(-EINVAL); 2989176e21eeSChuck Lever out_err: 2990315f3812SKinglong Mee xs_xprt_free(xprt); 2991176e21eeSChuck Lever return ret; 2992176e21eeSChuck Lever } 2993176e21eeSChuck Lever 29942881ae74STrond Myklebust static const struct rpc_timeout xs_udp_default_timeout = { 29952881ae74STrond Myklebust .to_initval = 5 * HZ, 29962881ae74STrond Myklebust .to_maxval = 30 * HZ, 29972881ae74STrond Myklebust .to_increment = 5 * HZ, 29982881ae74STrond Myklebust .to_retries = 5, 29992881ae74STrond Myklebust }; 30002881ae74STrond Myklebust 30019903cd1cSChuck Lever /** 30029903cd1cSChuck Lever * xs_setup_udp - Set up transport to use a UDP socket 300396802a09SFrank van Maarseveen * @args: rpc transport creation arguments 30049903cd1cSChuck Lever * 30059903cd1cSChuck Lever */ 3006483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_udp(struct xprt_create *args) 3007a246b010SChuck Lever { 30088f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 3009c8541ecdSChuck Lever struct rpc_xprt *xprt; 3010c8475461SChuck Lever struct sock_xprt *transport; 30110a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 3012a246b010SChuck Lever 3013d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_udp_slot_table_entries, 3014d9ba131dSTrond Myklebust xprt_udp_slot_table_entries); 3015c8541ecdSChuck Lever if (IS_ERR(xprt)) 3016c8541ecdSChuck Lever return xprt; 3017c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 3018a246b010SChuck Lever 3019ec739ef0SChuck Lever xprt->prot = IPPROTO_UDP; 3020808012fbSChuck Lever xprt->tsh_size = 0; 3021a246b010SChuck Lever /* XXX: header size can vary due to auth type, IPv6, etc. */ 3022a246b010SChuck Lever xprt->max_payload = (1U << 16) - (MAX_HEADER << 3); 3023a246b010SChuck Lever 302403bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 302503bf4b70SChuck Lever xprt->reestablish_timeout = XS_UDP_REEST_TO; 302603bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 3027a246b010SChuck Lever 3028262965f5SChuck Lever xprt->ops = &xs_udp_ops; 3029a246b010SChuck Lever 3030ba7392bbSTrond Myklebust xprt->timeout = &xs_udp_default_timeout; 3031a246b010SChuck Lever 3032f9b2ee71STrond Myklebust INIT_WORK(&transport->recv_worker, xs_udp_data_receive_workfn); 3033edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_udp_setup_socket); 3034edc1b01cSTrond Myklebust 30358f9d5b1aSChuck Lever switch (addr->sa_family) { 30368f9d5b1aSChuck Lever case AF_INET: 30378f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 30388f9d5b1aSChuck Lever xprt_set_bound(xprt); 30398f9d5b1aSChuck Lever 30409dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP); 30418f9d5b1aSChuck Lever break; 30428f9d5b1aSChuck Lever case AF_INET6: 30438f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 30448f9d5b1aSChuck Lever xprt_set_bound(xprt); 30458f9d5b1aSChuck Lever 30469dc3b095SChuck Lever xs_format_peer_addresses(xprt, "udp", RPCBIND_NETID_UDP6); 30478f9d5b1aSChuck Lever break; 30488f9d5b1aSChuck Lever default: 30490a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 30500a68b0beSJ. Bruce Fields goto out_err; 30518f9d5b1aSChuck Lever } 30528f9d5b1aSChuck Lever 3053c740eff8SChuck Lever if (xprt_bound(xprt)) 3054c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3055c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 3056c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 3057c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 3058c740eff8SChuck Lever else 3059c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 3060c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 3061c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 3062edb267a6SChuck Lever 3063bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 3064c8541ecdSChuck Lever return xprt; 30650a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 30660a68b0beSJ. Bruce Fields out_err: 3067315f3812SKinglong Mee xs_xprt_free(xprt); 30680a68b0beSJ. Bruce Fields return ret; 3069a246b010SChuck Lever } 3070a246b010SChuck Lever 30712881ae74STrond Myklebust static const struct rpc_timeout xs_tcp_default_timeout = { 30722881ae74STrond Myklebust .to_initval = 60 * HZ, 30732881ae74STrond Myklebust .to_maxval = 60 * HZ, 30742881ae74STrond Myklebust .to_retries = 2, 30752881ae74STrond Myklebust }; 30762881ae74STrond Myklebust 30779903cd1cSChuck Lever /** 30789903cd1cSChuck Lever * xs_setup_tcp - Set up transport to use a TCP socket 307996802a09SFrank van Maarseveen * @args: rpc transport creation arguments 30809903cd1cSChuck Lever * 30819903cd1cSChuck Lever */ 3082483066d6SAdrian Bunk static struct rpc_xprt *xs_setup_tcp(struct xprt_create *args) 3083a246b010SChuck Lever { 30848f9d5b1aSChuck Lever struct sockaddr *addr = args->dstaddr; 3085c8541ecdSChuck Lever struct rpc_xprt *xprt; 3086c8475461SChuck Lever struct sock_xprt *transport; 30870a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 3088b7993cebSTrond Myklebust unsigned int max_slot_table_size = xprt_max_tcp_slot_table_entries; 3089b7993cebSTrond Myklebust 3090b7993cebSTrond Myklebust if (args->flags & XPRT_CREATE_INFINITE_SLOTS) 3091b7993cebSTrond Myklebust max_slot_table_size = RPC_MAX_SLOT_TABLE_LIMIT; 3092a246b010SChuck Lever 3093d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 3094b7993cebSTrond Myklebust max_slot_table_size); 3095c8541ecdSChuck Lever if (IS_ERR(xprt)) 3096c8541ecdSChuck Lever return xprt; 3097c8475461SChuck Lever transport = container_of(xprt, struct sock_xprt, xprt); 3098a246b010SChuck Lever 3099ec739ef0SChuck Lever xprt->prot = IPPROTO_TCP; 3100808012fbSChuck Lever xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 3101808012fbSChuck Lever xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 3102a246b010SChuck Lever 310303bf4b70SChuck Lever xprt->bind_timeout = XS_BIND_TO; 310403bf4b70SChuck Lever xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO; 310503bf4b70SChuck Lever xprt->idle_timeout = XS_IDLE_DISC_TO; 3106a246b010SChuck Lever 3107262965f5SChuck Lever xprt->ops = &xs_tcp_ops; 3108ba7392bbSTrond Myklebust xprt->timeout = &xs_tcp_default_timeout; 3109a246b010SChuck Lever 31103851f1cdSTrond Myklebust xprt->max_reconnect_timeout = xprt->timeout->to_maxval; 31117196dbb0STrond Myklebust xprt->connect_timeout = xprt->timeout->to_initval * 31127196dbb0STrond Myklebust (xprt->timeout->to_retries + 1); 31133851f1cdSTrond Myklebust 3114edc1b01cSTrond Myklebust INIT_WORK(&transport->recv_worker, xs_tcp_data_receive_workfn); 3115edc1b01cSTrond Myklebust INIT_DELAYED_WORK(&transport->connect_worker, xs_tcp_setup_socket); 3116edc1b01cSTrond Myklebust 31178f9d5b1aSChuck Lever switch (addr->sa_family) { 31188f9d5b1aSChuck Lever case AF_INET: 31198f9d5b1aSChuck Lever if (((struct sockaddr_in *)addr)->sin_port != htons(0)) 31208f9d5b1aSChuck Lever xprt_set_bound(xprt); 31218f9d5b1aSChuck Lever 31229dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP); 31238f9d5b1aSChuck Lever break; 31248f9d5b1aSChuck Lever case AF_INET6: 31258f9d5b1aSChuck Lever if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0)) 31268f9d5b1aSChuck Lever xprt_set_bound(xprt); 31278f9d5b1aSChuck Lever 31289dc3b095SChuck Lever xs_format_peer_addresses(xprt, "tcp", RPCBIND_NETID_TCP6); 31298f9d5b1aSChuck Lever break; 31308f9d5b1aSChuck Lever default: 31310a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 31320a68b0beSJ. Bruce Fields goto out_err; 31338f9d5b1aSChuck Lever } 31348f9d5b1aSChuck Lever 3135c740eff8SChuck Lever if (xprt_bound(xprt)) 3136c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3137c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 3138c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PORT], 3139c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 3140c740eff8SChuck Lever else 3141c740eff8SChuck Lever dprintk("RPC: set up xprt to %s (autobind) via %s\n", 3142c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_ADDR], 3143c740eff8SChuck Lever xprt->address_strings[RPC_DISPLAY_PROTO]); 3144c740eff8SChuck Lever 3145bc25571eS\"Talpey, Thomas\ if (try_module_get(THIS_MODULE)) 3146c8541ecdSChuck Lever return xprt; 31470a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 31480a68b0beSJ. Bruce Fields out_err: 3149315f3812SKinglong Mee xs_xprt_free(xprt); 31500a68b0beSJ. Bruce Fields return ret; 3151a246b010SChuck Lever } 3152282b32e1SChuck Lever 3153f300babaSAlexandros Batsakis /** 3154f300babaSAlexandros Batsakis * xs_setup_bc_tcp - Set up transport to use a TCP backchannel socket 3155f300babaSAlexandros Batsakis * @args: rpc transport creation arguments 3156f300babaSAlexandros Batsakis * 3157f300babaSAlexandros Batsakis */ 3158f300babaSAlexandros Batsakis static struct rpc_xprt *xs_setup_bc_tcp(struct xprt_create *args) 3159f300babaSAlexandros Batsakis { 3160f300babaSAlexandros Batsakis struct sockaddr *addr = args->dstaddr; 3161f300babaSAlexandros Batsakis struct rpc_xprt *xprt; 3162f300babaSAlexandros Batsakis struct sock_xprt *transport; 3163f300babaSAlexandros Batsakis struct svc_sock *bc_sock; 31640a68b0beSJ. Bruce Fields struct rpc_xprt *ret; 3165f300babaSAlexandros Batsakis 3166d9ba131dSTrond Myklebust xprt = xs_setup_xprt(args, xprt_tcp_slot_table_entries, 3167d9ba131dSTrond Myklebust xprt_tcp_slot_table_entries); 3168f300babaSAlexandros Batsakis if (IS_ERR(xprt)) 3169f300babaSAlexandros Batsakis return xprt; 3170f300babaSAlexandros Batsakis transport = container_of(xprt, struct sock_xprt, xprt); 3171f300babaSAlexandros Batsakis 3172f300babaSAlexandros Batsakis xprt->prot = IPPROTO_TCP; 3173f300babaSAlexandros Batsakis xprt->tsh_size = sizeof(rpc_fraghdr) / sizeof(u32); 3174f300babaSAlexandros Batsakis xprt->max_payload = RPC_MAX_FRAGMENT_SIZE; 3175f300babaSAlexandros Batsakis xprt->timeout = &xs_tcp_default_timeout; 3176f300babaSAlexandros Batsakis 3177f300babaSAlexandros Batsakis /* backchannel */ 3178f300babaSAlexandros Batsakis xprt_set_bound(xprt); 3179f300babaSAlexandros Batsakis xprt->bind_timeout = 0; 3180f300babaSAlexandros Batsakis xprt->reestablish_timeout = 0; 3181f300babaSAlexandros Batsakis xprt->idle_timeout = 0; 3182f300babaSAlexandros Batsakis 3183f300babaSAlexandros Batsakis xprt->ops = &bc_tcp_ops; 3184f300babaSAlexandros Batsakis 3185f300babaSAlexandros Batsakis switch (addr->sa_family) { 3186f300babaSAlexandros Batsakis case AF_INET: 3187f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3188f300babaSAlexandros Batsakis RPCBIND_NETID_TCP); 3189f300babaSAlexandros Batsakis break; 3190f300babaSAlexandros Batsakis case AF_INET6: 3191f300babaSAlexandros Batsakis xs_format_peer_addresses(xprt, "tcp", 3192f300babaSAlexandros Batsakis RPCBIND_NETID_TCP6); 3193f300babaSAlexandros Batsakis break; 3194f300babaSAlexandros Batsakis default: 31950a68b0beSJ. Bruce Fields ret = ERR_PTR(-EAFNOSUPPORT); 31960a68b0beSJ. Bruce Fields goto out_err; 3197f300babaSAlexandros Batsakis } 3198f300babaSAlexandros Batsakis 3199f300babaSAlexandros Batsakis dprintk("RPC: set up xprt to %s (port %s) via %s\n", 3200f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_ADDR], 3201f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PORT], 3202f300babaSAlexandros Batsakis xprt->address_strings[RPC_DISPLAY_PROTO]); 3203f300babaSAlexandros Batsakis 3204f300babaSAlexandros Batsakis /* 320599de8ea9SJ. Bruce Fields * Once we've associated a backchannel xprt with a connection, 320628303ca3SWeng Meiling * we want to keep it around as long as the connection lasts, 320728303ca3SWeng Meiling * in case we need to start using it for a backchannel again; 320828303ca3SWeng Meiling * this reference won't be dropped until bc_xprt is destroyed. 320999de8ea9SJ. Bruce Fields */ 321099de8ea9SJ. Bruce Fields xprt_get(xprt); 321199de8ea9SJ. Bruce Fields args->bc_xprt->xpt_bc_xprt = xprt; 321299de8ea9SJ. Bruce Fields xprt->bc_xprt = args->bc_xprt; 321399de8ea9SJ. Bruce Fields bc_sock = container_of(args->bc_xprt, struct svc_sock, sk_xprt); 321499de8ea9SJ. Bruce Fields transport->sock = bc_sock->sk_sock; 321599de8ea9SJ. Bruce Fields transport->inet = bc_sock->sk_sk; 321699de8ea9SJ. Bruce Fields 321799de8ea9SJ. Bruce Fields /* 3218f300babaSAlexandros Batsakis * Since we don't want connections for the backchannel, we set 3219f300babaSAlexandros Batsakis * the xprt status to connected 3220f300babaSAlexandros Batsakis */ 3221f300babaSAlexandros Batsakis xprt_set_connected(xprt); 3222f300babaSAlexandros Batsakis 3223f300babaSAlexandros Batsakis if (try_module_get(THIS_MODULE)) 3224f300babaSAlexandros Batsakis return xprt; 3225642aab58SKinglong Mee 3226642aab58SKinglong Mee args->bc_xprt->xpt_bc_xprt = NULL; 322739a9beabSJ. Bruce Fields args->bc_xprt->xpt_bc_xps = NULL; 322899de8ea9SJ. Bruce Fields xprt_put(xprt); 32290a68b0beSJ. Bruce Fields ret = ERR_PTR(-EINVAL); 32300a68b0beSJ. Bruce Fields out_err: 3231315f3812SKinglong Mee xs_xprt_free(xprt); 32320a68b0beSJ. Bruce Fields return ret; 3233f300babaSAlexandros Batsakis } 3234f300babaSAlexandros Batsakis 3235176e21eeSChuck Lever static struct xprt_class xs_local_transport = { 3236176e21eeSChuck Lever .list = LIST_HEAD_INIT(xs_local_transport.list), 3237176e21eeSChuck Lever .name = "named UNIX socket", 3238176e21eeSChuck Lever .owner = THIS_MODULE, 3239176e21eeSChuck Lever .ident = XPRT_TRANSPORT_LOCAL, 3240176e21eeSChuck Lever .setup = xs_setup_local, 3241176e21eeSChuck Lever }; 3242176e21eeSChuck Lever 3243bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_udp_transport = { 3244bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_udp_transport.list), 3245bc25571eS\"Talpey, Thomas\ .name = "udp", 3246bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3247f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_UDP, 3248bc25571eS\"Talpey, Thomas\ .setup = xs_setup_udp, 3249bc25571eS\"Talpey, Thomas\ }; 3250bc25571eS\"Talpey, Thomas\ 3251bc25571eS\"Talpey, Thomas\ static struct xprt_class xs_tcp_transport = { 3252bc25571eS\"Talpey, Thomas\ .list = LIST_HEAD_INIT(xs_tcp_transport.list), 3253bc25571eS\"Talpey, Thomas\ .name = "tcp", 3254bc25571eS\"Talpey, Thomas\ .owner = THIS_MODULE, 3255f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_TCP, 3256bc25571eS\"Talpey, Thomas\ .setup = xs_setup_tcp, 3257bc25571eS\"Talpey, Thomas\ }; 3258bc25571eS\"Talpey, Thomas\ 3259f300babaSAlexandros Batsakis static struct xprt_class xs_bc_tcp_transport = { 3260f300babaSAlexandros Batsakis .list = LIST_HEAD_INIT(xs_bc_tcp_transport.list), 3261f300babaSAlexandros Batsakis .name = "tcp NFSv4.1 backchannel", 3262f300babaSAlexandros Batsakis .owner = THIS_MODULE, 3263f300babaSAlexandros Batsakis .ident = XPRT_TRANSPORT_BC_TCP, 3264f300babaSAlexandros Batsakis .setup = xs_setup_bc_tcp, 3265f300babaSAlexandros Batsakis }; 3266f300babaSAlexandros Batsakis 3267282b32e1SChuck Lever /** 3268bc25571eS\"Talpey, Thomas\ * init_socket_xprt - set up xprtsock's sysctls, register with RPC client 3269282b32e1SChuck Lever * 3270282b32e1SChuck Lever */ 3271282b32e1SChuck Lever int init_socket_xprt(void) 3272282b32e1SChuck Lever { 3273f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 32742b1bec5fSEric W. Biederman if (!sunrpc_table_header) 32750b4d4147SEric W. Biederman sunrpc_table_header = register_sysctl_table(sunrpc_table); 3276fbf76683SChuck Lever #endif 3277fbf76683SChuck Lever 3278176e21eeSChuck Lever xprt_register_transport(&xs_local_transport); 3279bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_udp_transport); 3280bc25571eS\"Talpey, Thomas\ xprt_register_transport(&xs_tcp_transport); 3281f300babaSAlexandros Batsakis xprt_register_transport(&xs_bc_tcp_transport); 3282bc25571eS\"Talpey, Thomas\ 3283282b32e1SChuck Lever return 0; 3284282b32e1SChuck Lever } 3285282b32e1SChuck Lever 3286282b32e1SChuck Lever /** 3287bc25571eS\"Talpey, Thomas\ * cleanup_socket_xprt - remove xprtsock's sysctls, unregister 3288282b32e1SChuck Lever * 3289282b32e1SChuck Lever */ 3290282b32e1SChuck Lever void cleanup_socket_xprt(void) 3291282b32e1SChuck Lever { 3292f895b252SJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) 3293fbf76683SChuck Lever if (sunrpc_table_header) { 3294fbf76683SChuck Lever unregister_sysctl_table(sunrpc_table_header); 3295fbf76683SChuck Lever sunrpc_table_header = NULL; 3296fbf76683SChuck Lever } 3297fbf76683SChuck Lever #endif 3298bc25571eS\"Talpey, Thomas\ 3299176e21eeSChuck Lever xprt_unregister_transport(&xs_local_transport); 3300bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_udp_transport); 3301bc25571eS\"Talpey, Thomas\ xprt_unregister_transport(&xs_tcp_transport); 3302f300babaSAlexandros Batsakis xprt_unregister_transport(&xs_bc_tcp_transport); 3303282b32e1SChuck Lever } 3304cbf11071STrond Myklebust 33059bbb9e5aSRusty Russell static int param_set_uint_minmax(const char *val, 33069bbb9e5aSRusty Russell const struct kernel_param *kp, 3307cbf11071STrond Myklebust unsigned int min, unsigned int max) 3308cbf11071STrond Myklebust { 330900cfaa94SDaniel Walter unsigned int num; 3310cbf11071STrond Myklebust int ret; 3311cbf11071STrond Myklebust 3312cbf11071STrond Myklebust if (!val) 3313cbf11071STrond Myklebust return -EINVAL; 331400cfaa94SDaniel Walter ret = kstrtouint(val, 0, &num); 33159761a246SDan Carpenter if (ret) 33169761a246SDan Carpenter return ret; 33179761a246SDan Carpenter if (num < min || num > max) 3318cbf11071STrond Myklebust return -EINVAL; 3319cbf11071STrond Myklebust *((unsigned int *)kp->arg) = num; 3320cbf11071STrond Myklebust return 0; 3321cbf11071STrond Myklebust } 3322cbf11071STrond Myklebust 33239bbb9e5aSRusty Russell static int param_set_portnr(const char *val, const struct kernel_param *kp) 3324cbf11071STrond Myklebust { 3325ffb6ca33SFrank Sorenson if (kp->arg == &xprt_min_resvport) 3326cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3327cbf11071STrond Myklebust RPC_MIN_RESVPORT, 3328ffb6ca33SFrank Sorenson xprt_max_resvport); 3329ffb6ca33SFrank Sorenson return param_set_uint_minmax(val, kp, 3330ffb6ca33SFrank Sorenson xprt_min_resvport, 3331cbf11071STrond Myklebust RPC_MAX_RESVPORT); 3332cbf11071STrond Myklebust } 3333cbf11071STrond Myklebust 33349c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_portnr = { 33359bbb9e5aSRusty Russell .set = param_set_portnr, 33369bbb9e5aSRusty Russell .get = param_get_uint, 33379bbb9e5aSRusty Russell }; 33389bbb9e5aSRusty Russell 3339cbf11071STrond Myklebust #define param_check_portnr(name, p) \ 3340cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3341cbf11071STrond Myklebust 3342cbf11071STrond Myklebust module_param_named(min_resvport, xprt_min_resvport, portnr, 0644); 3343cbf11071STrond Myklebust module_param_named(max_resvport, xprt_max_resvport, portnr, 0644); 3344cbf11071STrond Myklebust 33459bbb9e5aSRusty Russell static int param_set_slot_table_size(const char *val, 33469bbb9e5aSRusty Russell const struct kernel_param *kp) 3347cbf11071STrond Myklebust { 3348cbf11071STrond Myklebust return param_set_uint_minmax(val, kp, 3349cbf11071STrond Myklebust RPC_MIN_SLOT_TABLE, 3350cbf11071STrond Myklebust RPC_MAX_SLOT_TABLE); 3351cbf11071STrond Myklebust } 3352cbf11071STrond Myklebust 33539c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_slot_table_size = { 33549bbb9e5aSRusty Russell .set = param_set_slot_table_size, 33559bbb9e5aSRusty Russell .get = param_get_uint, 33569bbb9e5aSRusty Russell }; 33579bbb9e5aSRusty Russell 3358cbf11071STrond Myklebust #define param_check_slot_table_size(name, p) \ 3359cbf11071STrond Myklebust __param_check(name, p, unsigned int); 3360cbf11071STrond Myklebust 3361d9ba131dSTrond Myklebust static int param_set_max_slot_table_size(const char *val, 3362d9ba131dSTrond Myklebust const struct kernel_param *kp) 3363d9ba131dSTrond Myklebust { 3364d9ba131dSTrond Myklebust return param_set_uint_minmax(val, kp, 3365d9ba131dSTrond Myklebust RPC_MIN_SLOT_TABLE, 3366d9ba131dSTrond Myklebust RPC_MAX_SLOT_TABLE_LIMIT); 3367d9ba131dSTrond Myklebust } 3368d9ba131dSTrond Myklebust 33699c27847dSLuis R. Rodriguez static const struct kernel_param_ops param_ops_max_slot_table_size = { 3370d9ba131dSTrond Myklebust .set = param_set_max_slot_table_size, 3371d9ba131dSTrond Myklebust .get = param_get_uint, 3372d9ba131dSTrond Myklebust }; 3373d9ba131dSTrond Myklebust 3374d9ba131dSTrond Myklebust #define param_check_max_slot_table_size(name, p) \ 3375d9ba131dSTrond Myklebust __param_check(name, p, unsigned int); 3376d9ba131dSTrond Myklebust 3377cbf11071STrond Myklebust module_param_named(tcp_slot_table_entries, xprt_tcp_slot_table_entries, 3378cbf11071STrond Myklebust slot_table_size, 0644); 3379d9ba131dSTrond Myklebust module_param_named(tcp_max_slot_table_entries, xprt_max_tcp_slot_table_entries, 3380d9ba131dSTrond Myklebust max_slot_table_size, 0644); 3381cbf11071STrond Myklebust module_param_named(udp_slot_table_entries, xprt_udp_slot_table_entries, 3382cbf11071STrond Myklebust slot_table_size, 0644); 3383